\savestack{\lstmfull}{\input{modules/Module3/tikz_images/lstm_full.tex}}
\savestack{\lstmarch}{\input{modules/Module3/tikz_images/lstm_arch.tex}}
\savestack{\lstmvanish}{\input{modules/Module3/tikz_images/lstm_arch_vanish.tex}}
\savestack{\lstmexplode}{\input{modules/Module3/tikz_images/lstm_arch_explode.tex}}

\begin{frame}
  \myheading{Module 15.3: How LSTMs avoid the problem of vanishing gradients}
\end{frame}

\begin{frame}
\begin{overlayarea}{\textwidth}{0.46\textheight}
	\begin{center}
		\makebox[\textwidth][l]{\usebox{\lstmfullcontent}}
	\end{center}

\end{overlayarea}

\begin{overlayarea}{\textwidth}{0.54\textheight}
	\textbf{Intuition}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}%\justifying
				\item<1-> During forward propagation the gates control the flow of information
				\item<2-> They prevent any irrelevant information from being written to the state
			\end{itemize}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}%\justifying
				\item<3-> Similarly during backward propagation they control the flow of gradients
				\item<4-> It is easy to see that during backward pass the gradients will get multiplied by the gate
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{overlayarea}
\end{frame}


%\begin{frame}
%  \begin{columns}
%    \column{0.5\textwidth}
%      \begin{overlayarea}{\textwidth}{\textheight}
%        \makebox[\textwidth][c]{\usebox{\lstmfullcontent}}
%      \end{overlayarea}
%
%    \column{0.5\textwidth}
%      \begin{overlayarea}{\textwidth}{\textheight}
%      \textbf{Intuition}
%      \begin{itemize}\justifying
%        \item<1-> During forward propagation the gates control the flow of information
%        \item<2-> They prevent any irrelevant information from being written to the state
%        \item<3-> Similarly during backward propagation they control the flow of gradients
%        \item<4-> It is easy to see that during backward pass the gradients will get multiplied by the gate
%      \end{itemize}
%      \end{overlayarea}
%  \end{columns}
%\end{frame}

\begin{frame}
\begin{overlayarea}{\textwidth}{0.4\textheight}
	\begin{center}
		\makebox[\textwidth][l]{\usebox{\lstmfullcontent}}
	\end{center}

\end{overlayarea}

\begin{overlayarea}{\textwidth}{0.6\textheight}
	\begin{itemize}\justifying
		\item<1-> If the state at time $t-1$ did not contribute much to the state at time $t$ (i.e., if $\|f_t\| \rightarrow 0$ and $\|o_{t-1}\|  \rightarrow 0$) then during backpropagation the gradients flowing into $s_{t-1}$ will vanish
		\item<2-> But this kind of a vanishing gradient is fine (since $s_{t-1}$ did not contribute to $s_{t}$ we don't want to hold it responsible for the crimes of $s_{t}$)
		\item<3-> The key difference from vanilla RNNs is that the flow of information and gradients is controlled by the gates which ensure that the gradients vanish only when they should (i.e., when $s_{t-1}$  didn't contribute much to $s_{t}$)
	\end{itemize}
\end{overlayarea}
\end{frame}

%\begin{frame}
%  \begin{columns}
%    \column{0.5\textwidth}
%      \begin{overlayarea}{\textwidth}{\textheight}
%        \makebox[\textwidth][c]{\usebox{\lstmfullcontent}}
%      \end{overlayarea}
%
%    \column{0.5\textwidth}
%      \begin{overlayarea}{\textwidth}{\textheight}
%      \textbf{Intuition}
%      \begin{itemize}\justifying
%        \item<1-> If the state at time $t-1$ did not contribute much to the state at time $t$ (i.e., if $\|f_t\| \rightarrow 0$ and $\|o_{t-1}\|  \rightarrow 0$) then during backpropagation the gradients flowing into $s_{t-1}$ will vanish
%        \item<2-> But this kind of a vanishing gradient is fine (since $s_{t-1}$ did not contribute to $s_{t}$ we don't want to hold it responsible for the crimes of $s_{t}$)
%        \item<3-> The key difference from vanilla RNNs is that the flow of information and gradients is controlled by the gates which ensure that the gradients vanish only when they should (i.e., when $s_{t-1}$  didn't contribute much to $s_{t}$)
%      \end{itemize}
%      \end{overlayarea}
%  \end{columns}
%\end{frame}

\begin{frame}
We will now see an illustrative proof of how the gates control the flow of gradients
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        %\makebox[\textwidth][c]{\usebox{\lstmfullcontent}}
        %the ordered network for RNN
        \input{modules/Module3/tikz_images/de_graph.tex}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      \begin{itemize}\justifying
        \item<1-> Recall that RNNs had this multiplicative term which caused the gradients to vanish
        \begin{align*}
          \frac{\partial \mathscr{L}_t(\theta)}{\partial W} &=
          \frac{\partial \mathscr{L}_t(\theta)}{\partial s_t}
          \sum_{k=1}^t
          \textcolor{red}{\prod_{j=k}^{t-1} \frac{\partial s_{j+1}}{\partial s_j}}
          \frac{\partial^+ s_k}{\partial W}
        \end{align*}
        \item<2-> In particular, if the loss at $\mathscr{L}_4(\theta)$ was high because $W$ was not good enough to compute $s_1$ correctly then this information will not be propagated back to $W$ as the gradient $\frac{\partial \mathscr{L}_t(\theta)}{\partial W}$ along this long path will vanish
      \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        %\makebox[\textwidth][c]{\usebox{\lstmfullcontent}}
        %the ordered network for RNN
        \input{modules/Module3/tikz_images/de_graph.tex}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      \begin{itemize}\justifying
        \item<1-> In general, the gradient of $\mathscr{L}_t(\theta)$ w.r.t. $\theta_i$ vanishes when the gradients flowing through \textbf{each and every path} from $L_t(\theta)$ to $\theta_i$ vanish.
        \item<2-> On the other hand, the gradient of $\mathscr{L}_t(\theta)$ w.r.t. $\theta_i$ explodes when the gradient flowing through \textbf{at least one path} explodes.
        \item<3-> We will first argue that in the case of LSTMs there exists at least one path through which the gradients can flow effectively (and hence no vanishing gradients)
      \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}


% \begin{frame}
%   \begin{overlayarea}{\textwidth}{\textheight}
%     \begin{block} {LSTM architecture}

%     \end{block}
%   \end{overlayarea}
% \end{frame}

\begin{frame}
  \begin{columns}
    \column{0.4\textwidth}
      \input{modules/Module3/tikz_images/lstm_arch_intro.tex}
    \column{0.6\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}\justifying
            \item<1-> We will start with the dependency graph involving different variables in LSTMs
            \item<2-> Starting with the states at timestep $k-1$
            \begin{align*}
              \onslide<4->{o_k &= \sigma(W_oh_{k-1} + U_ox_k + b_o) \\}
            \end{align*}
            \item<5-> For simplicity we will omit the parameters for now and return back to  them later
            \begin{align*}
              \onslide<6->{i_k &= \sigma(W_ih_{k-1} + U_ix_k + b_i) \\}
              \onslide<6->{f_k &= \sigma(W_fh_{k-1} + U_fx_k + b_f) \\}
              \onslide<6->{\tilde{s_k} &= \sigma(Wh_{k-1} + Ux_{k} + b) \\}
              \onslide<7->{s_k &= f_k \odot s_{k-1} + i_k \odot \tilde{s_k} \\}
              \visible<8->{h_k &= o_k \odot \sigma(s_k)\\}
            \end{align*}
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.4\textwidth}
      \only<1-2>{\input{modules/Module3/tikz_images/lstm_arch_intro2.tex}}
      \only<3->{\input{modules/Module3/tikz_images/lstm_arch_intro3_params.tex}}
    \column{0.6\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}\justifying
            \item<1-> Starting from $h_{k-1}$ and $s_{k-1}$ we have reached $h_{k}$ and $s_{k}$
            \item<2-> And the recursion will now continue till the last timestep
            \only<3>{\item For simplicity and ease of illustration, instead of considering the parameters ($\color{red} W$, $\color{red} W_o$, $\color{red} W_i$, $\color{red} W_f$, $U$, $U_o$, $U_i$, $U_f$) as separate nodes in the graph we will just put them on the appropriate edges. (We show only a few parameters and not all)}
            \item<4-> For simplicity and ease of illustration, instead of considering the parameters ($ W$, $W_o$, $W_i$, $W_f$, $U$, $U_o$, $U_i$, $U_f$) as separate nodes in the graph we will just put them on the appropriate edges. (We show only a few parameters and not all)
            \item<4-> We are now interested in knowing if the gradient from $\mathscr{L}_t(\theta)$ flows back to an arbitrary timestep $k$
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.4\textwidth}
      %\only<1>{\input{modules/Module3/tikz_images/lstm_arch_params_hl.tex}}
      %\only<2->{\input{modules/Module3/tikz_images/lstm_arch.tex}}
      \input{modules/Module3/tikz_images/lstm_arch_params_hl.tex}
    \column{0.6\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}\justifying
            \item<1-> For example, we are interested in knowing if the gradient flows to $W_f$ through $s_k$
            \item<2-> In other words, if $\mathscr{L}_t(\theta)$ was high because $W_f$ failed to compute an appropriate value for $s_k$ then this information should flow back to $W_f$ through the gradients
            \item<3-> We can ask a similar question about the other parameters (for example, $W_i$, $W_o$, $W$, etc.)
            \item<4-> How does LSTM ensure that this gradient does not vanish even at arbitrary time steps? Let us see
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.4\textwidth}
      %\input{modules/Module3/tikz_images/lstm_arch_wf.tex}
      \input{modules/Module3/tikz_images/lstm_arch_params_hl.tex}
    \column{0.6\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}\justifying
            \item<1-> It is sufficient to show that $\pdv{\mathcal{L}_t(\theta)}{s_{k}}$ does not vanish (because if this does not vanish we can reach $W_f$ through $s_k$)
            \item<2-> First, we observe that there are multiple paths from $\mathscr{L}_t(\theta)$ to $s_k$ (you just need to reverse the direction of the arrows for backpropagation)
            \item<3-> For example, there is one path through $s_{k+1}$, another through $h_k$
            \item<4-> Further, there are multiple paths to reach to $h_k$ itself (as should be obvious from the number of outgoing arrows from $h_k$)
            \item<5-> So at this point just convince yourself that there are many paths from $\mathscr{L}_t(\theta)$ to $s_k$
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
  \column{0.4\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      %\only<1>{\makebox[\textwidth][c]{\usebox{\lstmarchcontent}}}
      %\only<2->{\makebox[\textwidth][c]{\usebox{\lstmvanishcontent}}}
      \makebox[\textwidth][c]{\usebox{\lstmvanishcontent}}
    \end{overlayarea}

  \column{0.6\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      % \begin{block} {LSTM solves vanishing gradients problem: Proof}
      \begin{itemize}\justifying\itemsep0mm
        \item<1-> Consider one such path (highlighted) which will contribute to the gradient
        \item<2-> Let us denote the gradient along this path as $t_0$
        \begin{align*}
          \visible<3->{  t_0 &= \color{red}\pdv{\mathscr{L}_t(\theta)}{h_t}\pdv{h_t}{s_t}
                                    \color{blue}  \pdv{s_t}{s_{t-1}} \hdots \pdv{s_{k+1}}{s_k}
                      }
          %\visible<4->{      &= \color{red} \mathcal{L}_t'(h_t).\mathcal{D}(o_t \odot \sigma'(s_t))
          %                          \color{blue}\mathcal{D}(f_t) \hdots \mathcal{D}(f_{k+1}) \mathcal{D}(f_{k})
          %            }\\
          %\visible<5->{      &= \color{red} \mathcal{L}_t'(h_t).\mathcal{D}(o_t \odot \sigma'(s_t))
          %                          \color{blue}\mathcal{D}(f_t \odot \hdots \odot f_{k+1} \odot f_k)
          %            }\\
          %\visible<6->{      &= \color{red} \mathcal{L}_t'(h_t).\mathcal{D}(o_t \odot \sigma'(s_t))
           %                         \color{blue} \mathcal{D}(\odot_{i=k}^{t} f_i)
           %           }
          \color{black}
        \end{align*}
        \item<4-> The first term $\color{red}\pdv{\mathscr{L}_t(\theta)}{h_t}$ \color{black} is fine and it doesn't vanish ($h_t$ is directly connected to $\mathscr{L}_t(\theta)$ and there are no intermediate nodes which can cause the gradient to vanish)
        \item<5-> We will now look at the other terms\\ $\color{red}\pdv{h_t}{s_t}$ $\color{blue}\pdv{s_t}{s_{t-1}}$ $\color{blue} (\forall t) $
      \end{itemize}
      % % \end{block}
    \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
  \column{0.4\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \makebox[\textwidth][c]{\usebox{\lstmvanishcontent}}
    \end{overlayarea}

  \column{0.6\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      % \begin{block} {LSTM solves vanishing gradients problem: Proof}
      \begin{itemize}\itemsep0mm
        \item<1-> Let us first look at $\color{red}\pdv{h_t}{s_t}$
        \item<2-> Recall that
        \begin{align*}
              \visible<2->{h_t &= o_t \odot \sigma(s_t)}
          \color{black}
        \end{align*}
        \item<3-> Note that $h_{ti}$ only depends on $o_{ti}$ and $s_{ti}$ and not on any other elements of $o_{t}$ and $s_{t}$
        \item<4-> $\color{red}\pdv{h_t}{s_t}$ \color{black} will thus be a square diagonal matrix $\in \mathbb{R}^{d \times d}$ whose diagonal will be $o_t \odot \sigma'(s_t) \in \mathbb{R}^d$ (see slide 35 of Lecture 14)
        \item<5-> We will represent this diagonal matrix by $\mathcal{D}(o_t \odot \sigma'(s_t))$
      \end{itemize}
      % % \end{block}
    \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
  \column{0.4\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \makebox[\textwidth][c]{\usebox{\lstmvanishcontent}}
    \end{overlayarea}

  \column{0.6\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      % \begin{block} {LSTM solves vanishing gradients problem: Proof}
      \begin{itemize}\justifying\itemsep0mm
        \item<1-> Now let us consider $\color{blue} \pdv{s_t}{s_{t-1}}$
        \item<2-> Recall that
        \begin{align*}
              \visible<2->{s_t &= f_t \odot s_{t-1} + i_t \odot \tilde{s_t}}
          \color{black}
        \end{align*}
        \item<3-> Notice that  $\tilde{s_t}$ also depends on $s_{t-1}$ so we cannot treat it as a constant
        \item<4-> So once again we are dealing with an ordered network and thus $\color{blue} \pdv{s_t}{s_{t-1}}$ will be a sum of an explicit term and an implicit term (see slide 37 from Lecture 14)
        \item<5-> For simplicity, let us assume that the gradient from the implicit term vanishes (we are assuming a worst case scenario)
        \item<6-> And the gradient from the explicit term (treating $\tilde{s_t}$ as a constant) is given by $\mathcal{D}(f_t)$
      \end{itemize}
      % % \end{block}
    \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
  \column{0.4\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \makebox[\textwidth][c]{\usebox{\lstmvanishcontent}}
    \end{overlayarea}

  \column{0.6\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      % \begin{block} {LSTM solves vanishing gradients problem: Proof}
      \begin{itemize}\justifying\itemsep0mm
        \item<1-> We now return back to our full expression for $t_0$:
        \begin{align*}
          \visible<2->{  t_0 &= \color{red}\pdv{\mathcal{L}_t(\theta)}{h_t}\pdv{h_t}{s_t}
                                    \color{blue}  \pdv{s_t}{s_{t-1}} \hdots \pdv{s_{k+1}}{s_k}
                      }\\
          \visible<3->{      &= \color{red} \mathcal{L}_t'(h_t).\mathcal{D}(o_t \odot \sigma'(s_t))
                                    \color{blue}\mathcal{D}(f_t) \hdots \mathcal{D}(f_{k+1})
                      }\\
          \visible<4->{      &= \color{red} \mathcal{L}_t'(h_t).\mathcal{D}(o_t \odot \sigma'(s_t))
                                    \color{blue}\mathcal{D}(f_t \odot \hdots \odot f_{k+1})
                      }\\
          \visible<5->{      &= \color{red} \mathcal{L}_t'(h_t).\mathcal{D}(o_t \odot \sigma'(s_t))
                                    \color{blue} \mathcal{D}(\odot_{i=k+1}^{t} f_i)
                      }
          \color{black}
        \end{align*}
        %\item<7-> Thus, we have one path through which the gradient does not vanish due to recursive matrix multiplications
        \item<6-> The red terms don't vanish and the blue terms contain a multiplication of the forget gates
        \item<6-> The forget gates thus regulate the gradient flow depending on the explicit contribution of a state ($s_t$) to the next state $s_{t+1}$
      \end{itemize}
      % % \end{block}
    \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
  \column{0.4\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \makebox[\textwidth][c]{\usebox{\lstmvanishcontent}}
    \end{overlayarea}

  \column{0.6\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      % \begin{block} {LSTM solves vanishing gradients problem: Proof}
      \begin{itemize}\justifying\itemsep0mm
        \item<1-> If during forward pass $s_t$ did not contribute much to $s_{t+1}$ (because $f_t \rightarrow 0$) then during backpropgation also the gradient will not reach $s_t$
        \item<2-> This is fine because if $s_t$ did not contribute much to $s_{t+1}$ then there is no reason to hold it responsible during backpropgation \onslide<3->{($f_t$ does the same regulation during forward pass and backward pass which is fair)}
        \item<4-> Thus there exists this one path along which the gradient doesn't vanish when it shouldn't
        \item<5-> And as argued as long as the gradient flows back to $W_f$ through one of the paths ($t_0$) through $s_k$ we are fine !
        \item<6-> Of course the gradient flows back only when required as regulated by $f_i$'s (but let me just say it one last time that \textit{this is fair})
      \end{itemize}
      % % \end{block}
    \end{overlayarea}
  \end{columns}
\end{frame}


\begin{frame}
  \begin{columns}
  \column{0.4\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \only<1-2>{\makebox[\textwidth][c]{\usebox{\lstmarchcontent}}}
      \only<3->{\makebox[\textwidth][c]{\usebox{\lstmexplodecontent}}}
    \end{overlayarea}

  \column{0.6\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
        \vspace{-3mm}
        \begin{itemize}\justifying
          \item<1-> Now we will see why LSTMs do not solve the problem of exploding gradients
          \item<2-> We will show a path through which the gradient can explode
          \item<3-> Let us compute one term (say $t_1$) of $\pdv{\mathcal{L}_t(\theta)}{h_{k-1}}$ corresponding to the highlighted path
          \small
          {
            \begin{align*}
              \visible<4->{t_1     =   &  \color{red}\pdv{\mathcal{L}_t(\theta)}{h_t}
                                          \color{blue} \left(\pdv{h_t}{o_t} \pdv{o_t}{h_{t-1}} \right) \hdots \left(\pdv{h_k}{o_k} \pdv{o_k}{h_{k-1}} \right)}\\
             %\begin{align*}
              \visible<5->{        =   &  \color{red}\mathcal{L}_t'(h_t)
                                          \color{blue} \left(\mathcal{D}(\sigma(s_t)\odot o_t').W_o\right) \hdots \\
                                        & ~~~~ \color{blue} \left(\mathcal{D}(\sigma(s_k)\odot o_k').W_o\right)
                                        }\\
              \visible<6->{\|t_1\| \leq&  \color{red} \|\mathcal{L}_t'(h_t)\| \color{blue} \left(\|K\|\|W_o\|\right)^{t-k+1} }
            \end{align*}
          }
          \item<7-> Depending on the norm of matrix $W_o$, the gradient $\pdv{\mathcal{L}_t(\theta)}{h_{k-1}}$ may explode
          \item<8-> Similarly, $W_i$, $W_f$ and $W$ can also cause the gradients to explode
        \end{itemize}
    \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.4\textwidth}
      \makebox[\textwidth][c]{\usebox{\lstmexplodecontent}}
    \column{0.6\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}\justifying
            \item<1-> So how do we deal with the problem of exploding gradients ?
            \item<2-> One popular trick is to use gradient clipping
            \item<3-> While backpropagating if the norm of the gradient exceeds a certain value, it is scaled to keep its norm within an acceptable threshold$^*$ \footnotetext{\hspace{-0.2in}$^*$Pascanu, Razvan, Tomas Mikolov, and Yoshua Bengio. \\``On the difficulty of training recurrent neural networks."\\   ICML(3)28(2013):1310-1318}
            \item<4-> Essentially we retain the direction of the gradient but scale down the norm
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\if 0
\begin{frame}
  \begin{overlayarea}{\textwidth}{\textheight}
    \begin{block} {Revisiting the problem of vanishing/exploding gradients in RNNs}
      \begin{itemize}\justifying
        \item<1-> Gradient of $L_t(\theta)$ wrt $\theta_i$ vanishes when the gradient flow through \textbf{all the paths} from $L_t(\theta)$ to $\theta_i$ vanishes.
        \item<2-> Gradient of $L_t(\theta)$ wrt $\theta_i$ explodes when the gradient flow through \textbf{atleast one path} explodes.
        \item<3-> We will show that in the case of LSTMs, there exists at least one path along which the gradient does not vanish
      \end{itemize}
    \end{block}
  \end{overlayarea}
\end{frame}

\begin{frame}
  %\begin{overlayarea}{\textwidth}{\textheight}
    \begin{block} {Do LSTMs address this issue?}
      \begin{itemize}\justifying
        \item<1-> \alert<3>{We will show one path through which gradient w.r.t weight does not vanish}
        \item<2-> Later, we will show one path through which gradient w.r.t weight may explode
      \end{itemize}
    \end{block}
  %\end{overlayarea}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.4\textwidth}
      \input{modules/Module3/tikz_images/lstm_arch_intro.tex}
    \column{0.6\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}
            \item<1-> \textbf{Notation:} $\mathcal{D}(x)$ represents a square matrix $A\in \mathbb{R}^{n}$ whose diagonal is $x \in \mathbb{R}^n$.
            \item<2-> \textbf{Dependency Graph:} \visible<3->{Starting with the states at timestep $k-1$}
            \begin{align*}
              \onslide<4->{o_k &= \sigma(W_oh_{k-1} + U_ox_k + b_o) \\}
              \onslide<5->{&\text{Omitting $W,U,b$ and focusing }\\}
              \onslide<5->{&\text{on recursive connections}\\}
              \onslide<6->{i_k &= \sigma(W_ih_{k-1} + U_ix_k + b_i) \\}
              \onslide<6->{f_k &= \sigma(W_fh_{k-1} + U_fx_k + b_f) \\}
              \onslide<6->{\tilde{s_k} &= \sigma(Wh_{k-1} + Ux_{k} + b) \\}
              \onslide<7->{s_k &= f_k \odot s_{k-1} + i_k \odot \tilde{s_k} \\}
              \visible<8->{h_k &= o_k \odot \sigma(s_k)\\}
            \end{align*}
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{itemize}
    \item<1-> We have shown that the gradient $\pdv{\mathcal{L}_t(\theta)}{s_{k-1}}$ does not vanish.
    \item<2-> We have also shown that the gradient  $\pdv{\mathcal{L}_t(\theta)}{h_{k-1}}$ may explode.
    \item<3-> Is that all? We are interested in the gradients w.r.t the weights and biases.
    \item<4-> Let us take the term of gradient of loss w.r.t a weight, say $W_i$, which is contributed by the LSTM cell at timestep $k$.
    \begin{align*}
      \visible<5->{\pdv{\mathcal{L}_t(\theta)}{W_i} &= \sum_{j=1}^{t} {\pdv{\mathcal{L}_t(\theta)}{s_j}} \pdv{s_j}{i_k} \pdv{i_k}{W_i} = \hdots + \alert<6-7>{\pdv{\mathcal{L}_t(\theta)}{s_k}} \pdv{s_k}{i_k} \pdv{i_k}{W_i} + \hdots}
    \end{align*}
    % \item[]<5-> We can show that this gradient does not vanish due to gradient through path via $s_k$:
    \item<6-> First term (highlighted) does not vanish unnecessarily, \visible<7->{thus LSTM cell at timestep $k$ will contribute appropriately to the gradient w.r.t. $W_i$, even when $k<<t$}
    \item<8-> Similarly, we can show that contribution of LSTM cell at timestep $k-1$ to the above gradient may explode due to path via $h_k$, as $k$ decreases
    \item<9-> Similar behaviour can be shown for other weights and biases too.
  \end{itemize}
\end{frame}
\begin{frame}
  \begin{block}{Avoiding Exploding Gradients problem}
  LSTMs solve vanishing gradients problem, but do not address exploding gradient problem. But exploding gradients is not such a big problem and can be avoided:
  \begin{itemize}
    \item<1-> \textbf{Gradient Clipping:} While backpropagating if norm of gradient goes above a certain value, it is truncated to keep it within certain values $^*$ \footnotetext{\visible<1->{$^*$Pascanu, Razvan, Tomas Mikolov, and Yoshua Bengio. ``On the difficulty of training recurrent neural networks." ICML (3) 28 (2013): 1310-1318}}
    % \item<2-> \textbf{Path Truncated BPTT:} Don't propagate gradient via bad paths (\textit{i.e.}, through which have recursive matrix multiplication)
    % \begin{itemize}
    %   \item<3-> Gradient flows back only via direct $s_{i+1}-s_{i}$ edges
    %   \item<4-> This would only be an approximation of the gradient. But using this method we can go back without vanishing and exploding gradient for a lot of timesteps.
    %   \item<5-> In last cell, where we want to compute gradients wrt gates and parameters, we take complete gradients through all paths in the cell.
    % \end{itemize}
  \end{itemize}
  \end{block}
\end{frame}


\fi
