\begin{frame}
	\myheading{Module 14.4: The problem of Exploding and Vanishing
Gradients}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      \begin{center}
      \include{de_graph}
      \end{center}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}\justifying
          \item<1-> We will now focus on $\frac{\partial s_t}{\partial s_k}$ and highlight an
                    important problem in training RNN's using BPTT
          \item[]<2->
                    \begin{align*}
                      \frac{\partial s_t}{\partial s_k} &=
                          \frac{\partial s_t}{\partial s_{t-1}}
                          \frac{\partial s_{t-1}}{\partial s_{t-2}}
                          \dots
                          \frac{\partial s_{k+1}}{\partial s_k}
                      \\
                      \only<3->{
                          &=
                          \prod_{j=k}^{t-1} \frac{\partial s_{j+1}}{\partial s_j}
                      }
                    \end{align*}
          \item<4-> Let us look at one such term in the product
                    (i.e., $\frac{\partial s_{j+1}}{\partial s_j}$)
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}



% \begin{frame}
%   \begin{columns}
%     \column{0.5\textwidth}
%       \begin{overlayarea}{\textwidth}{\textheight}
%         \onslide<4->{\begin{align*}
%                         a_j &= \left[ a_{j1}, a_{j2}, a_{j3}, \dots a_{jd},  \right]\\
%                 s_j &= \left[ \sigma(a_{j1}),\sigma(a_{j2}), \dots \sigma(a_{jd})  \right]
%                 \end{align*}}
%             \onslide<5->{
%                 \begin{align*}
%                 \frac{\partial s_{j}}{\partial a_{j}} &=
%                              \begin{bmatrix}}
% \onslide<6->{ \frac{\partial s_{j1}}{\partial a_{j1}}} &        \onslide<7->{ \frac{\partial s_{j2}}{\partial a_{j1}}} &        \onslide<8->{ \frac{\partial s_{j3}}{\partial a_{j1}}} &        \onslide<9->{ \dots }  \\
% \onslide<10->{ \frac{\partial s_{j1}}{\partial a_{j2}} & \frac{\partial s_{j2}}{\partial a_{j2}} &\ddots &  \\}
%  \onslide<11->{ \vdots & \vdots & \vdots & \frac{\partial s_{jd}}{\partial a_{jd}}}  \\
%                      \end{bmatrix}\\
% \onslide<12->{  &= \begin{bmatrix}
%                      \sigma^{'}(a_{j1}) & 0 & 0 & 0\\
%                     \onslide<13->{ 0 & \sigma^{'}(a_{j2}) & 0 & 0 \\
%                      0 & 0  & \ddots & \\
%                      0 & 0 & \dots & \sigma^{'}(a_{jd}) }
%                      \end{bmatrix}\\}
% \onslide<14->{  &= diag (\sigma^{'} (a_j))   }
%                 \end{align*}

%       \end{overlayarea}

%     \column{0.5\textwidth}
%     \vspace{-0.5cm}
%       \begin{overlayarea}{\textwidth}{\textheight}
%        \begin{itemize}
%           \item<1-> \onslide<1->{We are interested in $\frac{\partial s_j}{\partial s_{j-1}}$
%                         \vspace{-0.3cm}
%                    \onslide<2->{\begin{align*}
%                              a_j &= W s_{j-1} + b\\
%                              s_j &= \sigma(a_{j})
%                              \end{align*}}}

% \onslide<3->{   \vspace{-1cm}
%         \begin{align*}
%         \frac{\partial s_j}{\partial s_{j-1}} &= \frac{\partial s_j}{\partial a_{j}}           \frac{\partial a_j}{\partial s_{j-1}}\\
%  \onslide<12->{       &= diag(\sigma^{'}(a_{j})) W\\}
%       %\onslide<13->{  s_j &= \sigma(a_{j})
%        % \frac{\partial s_t}{\partial s_{k}}\\ &= \prod \frac{\partial s_j}{\partial s_{j-1}} \\  }
%        %\onslide<14->{ &= \prod diag(\sigma^{'}(a_{j})) W }
%         \end{align*}
%         %\vspace{-0.7cm}
%         }
%         \item<15-> We are interested in the magnitude of $\frac{\partial s_j}{\partial s_{j-1}} \leftarrow $ if it is small (large) $\frac{\partial s_t}{\partial s_{k}}$ and hence $\frac{\partial \mathscr{L}_t}{\partial W}$ will vanish (explode)
%         \end{itemize}
%       \end{overlayarea}
%   \end{columns}
% \end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
       \onslide<4->{ \begin{align*}
                       a_j &= \left[ a_{j1}, a_{j2}, a_{j3}, \dots a_{jd},  \right]\\
                       s_j &= \left[ \sigma(a_{j1}),\sigma(a_{j2}), \dots \sigma(a_{jd})  \right]
               \end{align*}}
            \vspace{-0.5cm}
            \begin{align*}
            \onslide<5->{
              \frac{\partial s_{j}}{\partial a_{j}} &=
                           \begin{bmatrix}
              \onslide<6->{ \frac{\partial s_{j1}}{\partial a_{j1}}} &        \onslide<7->{ \frac{\partial s_{j2}}{\partial a_{j1}}} &        \onslide<8->{ \frac{\partial s_{j3}}{\partial a_{j1}}} &        \onslide<9->{ \dots }  \\
              \onslide<10->{ \frac{\partial s_{j1}}{\partial a_{j2}} & \frac{\partial s_{j2}}{\partial a_{j2}} &\ddots &  \\}
              \onslide<11->{ \vdots & \vdots & \vdots & \frac{\partial s_{jd}}{\partial a_{jd}}}  \\
                   \end{bmatrix}\\}
              \onslide<12->{  &= \begin{bmatrix}
                   \sigma^{'}(a_{j1}) & 0 & 0 & 0\\
                  \onslide<13->{ 0 & \sigma^{'}(a_{j2}) & 0 & 0 \\
                   0 & 0  & \ddots & \\
                   0 & 0 & \dots & \sigma^{'}(a_{jd}) }
                   \end{bmatrix}\\}
              \onslide<14->{  &= diag (\sigma^{'} (a_j))   }
            \end{align*}

      \end{overlayarea}

    \column{0.5\textwidth}
    \vspace{-0.5cm}
      \begin{overlayarea}{\textwidth}{\textheight}
       \begin{itemize}
          \item<1-> We are interested in $\frac{\partial s_j}{\partial s_{j-1}}$
              \vspace{-0.3cm}
         \onslide<2->{\begin{align*}
                   a_j &= W s_j + b\\
                   s_j &= \sigma(a_{j})
                   \end{align*}}

\onslide<3->{   \vspace{-1cm}
        \begin{align*}
        \frac{\partial s_j}{\partial s_{j-1}} &= \frac{\partial s_j}{\partial a_{j}}           \frac{\partial a_j}{\partial s_{j-1}}\\
 \onslide<15->{       &= diag(\sigma^{'}(a_{j})) W\\}
      %\onslide<13->{  s_j &= \sigma(a_{j})
       % \frac{\partial s_t}{\partial s_{k}}\\ &= \prod \frac{\partial s_j}{\partial s_{j-1}} \\  }
       %\onslide<14->{ &= \prod diag(\sigma^{'}(a_{j})) W }
        \end{align*}
        %\vspace{-0.7cm}
        }
        \item<16-> We are interested in the magnitude of $\frac{\partial s_j}{\partial s_{j-1}} \leftarrow $ if it is small (large) $\frac{\partial s_t}{\partial s_{k}}$ and hence $\frac{\partial \mathscr{L}_t}{\partial W}$ will vanish (explode)
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
    \begin{columns}
        \column{0.5\textwidth}
        \begin{overlayarea}{\textwidth}{\textheight}
                    \begin{align*}
         \onslide<1->{           \norm{\frac{\partial s_j}{\partial s_{j-1}}} &= \norm{diag(\sigma^{'} (a_j)) W } \\}
        \onslide<2->{&\le \norm{diag(\sigma^{'} (a_j)} \norm{ W } }
        \end{align*}
        \onslide<3->{
        $\because \sigma(a_j)$ is a bounded function (sigmoid, tanh) $\sigma^{'}
        (a_j)$ is bounded}

        \begin{align*}
        \onslide<4->{\sigma^{'} (a_j) &\leq \frac{1}{4} = \gamma \left[ \text{if }\sigma\text{ is logistic }\right]\\}
        \onslide<5->{&\leq 1 = \gamma \left[ \text{if }\sigma\text{ is tanh }\right]\\}
        \onslide<6->{\norm{\frac{\partial s_j}{\partial s_{j-1}}} &\le \gamma \norm{W}\\
        &\le \gamma \lambda }
        \end{align*}

        \end{overlayarea}

        \column{0.5\textwidth}
        \begin{overlayarea}{\textwidth}{\textheight}
            \begin{align*}
             \onslide<7->{\norm{\frac{\partial s_t}{\partial s_{k}}} &= \norm{\prod_{j=k+1}^{t}  \frac{\partial s_{j}}{\partial s_{j-1}}} \\}
            \onslide<8->{&\le \prod_{j=k+1}^{t} \gamma \lambda \\}
            \onslide<9->{&\le (\gamma \lambda)^{t-k}}
            \end{align*}
            \begin{itemize}
                \item<10-> If $\gamma \lambda < 1$ the gradient will vanish
                \item<11-> If $\gamma \lambda > 1$ the gradient could explode
                \item<12-> This is known as the problem of vanishing/ exploding gradients
            \end{itemize}
        \end{overlayarea}
    \end{columns}
\end{frame}

\if 0
\begin{frame}

        \begin{block}{}
            \begin{itemize}
                \item Let us take a small detour and check the bound on $\norm{W}$

            \end{itemize}


        \end{block}


\end{frame}

\begin{frame}
    \begin{columns}
        \column{0.5\textwidth}
        \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}
            \item<7-> This product $\prod_{j=k}^{t-1} \frac{\partial s_{j+1}}{\partial s_{j}}$ over timesteps potentially makes RNN training unstable
        \end{itemize}
        \end{overlayarea}

        \column{0.5\textwidth}
        \begin{overlayarea}{\textwidth}{\textheight}

            \begin{align*}
             \norm{\frac{\partial s_t}{\partial s_{k}}} &= \norm{\prod_{j=k+1}^{t}  \frac{\partial s_{j}}{\partial s_{j-1}}} \\
            \onslide<2->{&\le \prod_{j=k+1}^{t} \gamma \lambda \\}
            \onslide<3->{&\le (\gamma \lambda)^{t-k}}
            \end{align*}
            \begin{itemize}
                \item<4-> If $\gamma \lambda < 1$ the gradient will vanish
                \item<5-> If $\gamma \lambda > 1$ the gradient could explode
                \item<6-> This is known as the problem of vanishing/ exploding gradients
            \end{itemize}


        \end{overlayarea}
    \end{columns}
\end{frame}
\fi
\begin{frame}
    \begin{columns}
        \column{0.5\textwidth}
        \begin{overlayarea}{\textwidth}{\textheight}
\vspace{1.5cm}
\begin{center}
    \begin{tikzpicture}
        \draw[draw=red!60,fill=red!20,thick,solid,rounded corners] (1.6,0.4) rectangle (4.5,4);
        \node at (4.1,3.7) {$L_t$};
    \onslide<1->{
        \node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (0,1){};
        \node at (0.5,1.2) {$w$};
        \node at (0.2,1.8) {$v$};
        \node at (0.2,0.2) {$u$};
        \node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (0,-0.5){};
        \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (0,-1){\tiny $x_1$};
        \draw[->](Aemb)--(A);
    }
    \onslide<1->{
        \node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (0,2.8){};
        \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (0,3.3){\tiny $y_1$};

        \draw[->](A)--(C);
    }


    \onslide<1->{
        \node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (1,1){};
        \node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (1,-0.5){};
        \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (1,-1){\tiny $x_2$};
        \draw[->](Aemb)--(A);
        \node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (1,2.8){};
        \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (1,3.3){\tiny $y_2$};
        \node at (1.5,1.2) {$w$};
        \node at (1.2,1.8) {$v$};
        \node at (1.2,0.2) {$u$};
        \draw[->](A)--(C);
    }
    \onslide<1->{
        \node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (2,1){};
        \node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (2,-0.5){};
        \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (2,-1){\tiny $x_3$};
        \draw[->](Aemb)--(A);
        \node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (2,2.8){};
        \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (2,3.3){\tiny $y_3$};
        \node at (2.5,1.2) {$w$};
        \node at (2.2,1.8) {$v$};
        \node at (2.2,0.2) {$u$};
        \draw[->](A)--(C);
    }
    \onslide<1->{
        \node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (3,1){};
        \node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (3,-0.5){};
        \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (3,-1){\tiny $x_4$};
        \draw[->](Aemb)--(A);
        \node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (3,2.8){};
        \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (3,3.3){\tiny $y_4$};
        \node at (3.5,1.2) {$w$};
        \node at (3.2,1.8) {$v$};
        \node at (3.2,0.2) {$u$};
        \draw[->](A)--(C);
    }
    \onslide<1->{
        \node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (4,1){};
        \node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (4,-0.5){};
        \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (4,-1){\tiny $x_n$};
        \draw[->](Aemb)--(A);
        \node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (4,2.8){};
        \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (4,3.3){\tiny $y_n$};
        \node at (3.5,1.2) {$w$};
        \node at (4.2,1.8) {$v$};
        \node at (4.2,0.2) {$u$};
        \draw[->](A)--(C);
        \draw[->](0.2,1)--(0.8,1);
        \draw[->](1.2,1)--(1.8,1);
        \draw[->](2.2,1)--(2.8,1);
        \draw[->](3.2,1)--(3.7,1);

    }
    \end{tikzpicture}

\end{center}

        \end{overlayarea}

        \column{0.5\textwidth}
        \begin{overlayarea}{\textwidth}{\textheight}

                \begin{itemize}
                \item<1-> One simple way of avoiding this is to use truncated backpropogation where we restrict the product to $\tau (< t - k) $ terms
                %\begin{align*}
                 %\onslide<2->{\frac{\partial s_t}{\partial s_k} & \approx \frac{\partial s_t}{\partial s_\tau}\\}

                 %\onslide<3->{\text{where } t-k &> t-\tau\\}
                 %\onslide<4->{\frac{\partial s_t}{\partial s_\tau} &= \prod_{j=\tau}^{t-1}  \frac{\partial s_{j+1}}{\partial s_{j}}}
                %\end{align*}
            \end{itemize}


        \end{overlayarea}
    \end{columns}
\end{frame}
