\begin{frame}
	\myheading{Module 14.3: Backpropagation through time}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \vspace{1.5cm}
      	      			\begin{center}
      	      				\begin{tikzpicture}
      	      				\onslide<1->{
      	      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (0,1){};
      	      					\node at (0.5,1.2) {\footnotesize $W$};
      	      					\node at (0.2,1.8) {\footnotesize $V$};
      	      					\node at (0.2,0.2) {\footnotesize $U$};
      	      					\node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (0,-0.5){};
      	      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (0,-1){\tiny $x_1$};
      	      					\draw[->](Aemb)--(A);
      	      				}
      	      				\onslide<1->{
      	      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (0,2.8){};
      	      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (0,3.3){\tiny $y_1$};

      	      					\draw[->](A)--(C);
      	      				}


      	      				\onslide<1->{
      	      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (1,1){};
      	      					\node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (1,-0.5){};
      	      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (1,-1){\tiny $x_2$};
      	      					\draw[->](Aemb)--(A);
      	      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (1,2.8){};
      	      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (1,3.3){\tiny $y_2$};
      	      					\node at (1.5,1.2) {\footnotesize $W$};
      	      					\node at (1.2,1.8) {\footnotesize $V$};
      	      					\node at (1.2,0.2) {\footnotesize $U$};
      	      					\draw[->](A)--(C);
      	      				}
                                \onslide<1->{
                                \node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (2,1){};
                                \node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (2,-0.5){};
                                \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (2,-1){\tiny $x_3$};
                                \draw[->](Aemb)--(A);
                                \node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (2,2.8){};
                                \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (2,3.3){\tiny $y_3$};
                                \node at (2.5,1.2) {\footnotesize $W$};
                                \node at (2.2,1.8) {\footnotesize $V$};
                                \node at (2.2,0.2) {\footnotesize $U$};
                                \draw[->](A)--(C);
                            }
                                \onslide<1->{
                                \node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (3,1){};
                                \node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (3,-0.5){};
                                \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (3,-1){\tiny $x_4$};
                                \draw[->](Aemb)--(A);
                                \node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (3,2.8){};
                                \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (3,3.3){\tiny $y_4$};
                                \node at (3.5,1.2) {\footnotesize $W$};
                                \node at (3.2,1.8) {\footnotesize $V$};
                                \node at (3.2,0.2) {\footnotesize $U$};
                                \draw[->](A)--(C);
                            }


                                \draw[->](0.15,1)--(0.85,1);
                                \draw[->](1.15,1)--(1.85,1);

                                \draw[->](2.15,1)--(2.85,1);
                                \draw[->](3.15,1)--(3.85,1);


      	      				\end{tikzpicture}

      	      			\end{center}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}\justifying
          \item<1-> Before proceeding let us look at the dimensions
          of the parameters carefully
            \begin{align*}
              \onslide<2-> {x_i &\in \mathbb{R}^n \text{\quad (n-dimensional input)}} \\
              \onslide<3-> {s_i &\in \mathbb{R}^d \text{\quad (d-dimensional state)}}\\
              \onslide<4-> {y_i &\in \mathbb{R}^k  \text{\quad (say k classes)}} \\
              \onslide<5-> {U &\in \onslide<6->{\mathbb{R}^{n \times d}}} \\
              \onslide<7-> {V &\in \onslide<8->{\mathbb{R}^{d \times k}}} \\
              \onslide<9-> {W &\in \onslide<10->{\mathbb{R}^{d \times d}}} \\
            \end{align*}
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      \vspace{1.5cm}
                \begin{center}
      	      	              \begin{tikzpicture}
                            \onslide<1->{
                                \node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (0,1){};
                                \node at (0.5,1.2) {\footnotesize $W$};
                                \node at (0.2,1.8) {\footnotesize $V$};
                                \node at (0.2,0.2) {\footnotesize $U$};
                                \node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (0,-0.5){};
                                \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (0,-1){\tiny $x_1$};
                                \draw[->](Aemb)--(A);
                            }
                            \onslide<1->{
                                \node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (0,2.8){};
                                \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (0,3.3){\tiny $y_1$};

                                \draw[->](A)--(C);
                            }


                            \onslide<1->{
                                \node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (1,1){};
                                \node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (1,-0.5){};
                                \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (1,-1){\tiny $x_2$};
                                \draw[->](Aemb)--(A);
                                \node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (1,2.8){};
                                \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (1,3.3){\tiny $y_2$};
                                \node at (1.5,1.2) {\footnotesize $W$};
                                \node at (1.2,1.8) {\footnotesize $V$};
                                \node at (1.2,0.2) {\footnotesize $U$};
                                \draw[->](A)--(C);
                            }
                                \onslide<1->{
                                \node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (2,1){};
                                \node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (2,-0.5){};
                                \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (2,-1){\tiny $x_3$};
                                \draw[->](Aemb)--(A);
                                \node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (2,2.8){};
                                \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (2,3.3){\tiny $y_3$};
                                \node at (2.5,1.2) {\footnotesize $W$};
                                \node at (2.2,1.8) {\footnotesize $V$};
                                \node at (2.2,0.2) {\footnotesize $U$};
                                \draw[->](A)--(C);
                            }
                                \onslide<1->{
                                \node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (3,1){};
                                \node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (3,-0.5){};
                                \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (3,-1){\tiny $x_4$};
                                \draw[->](Aemb)--(A);
                                \node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (3,2.8){};
                                \node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (3,3.3){\tiny $y_4$};
                                \node at (3.5,1.2) {\footnotesize $W$};
                                \node at (3.2,1.8) {\footnotesize $V$};
                                \node at (3.2,0.2) {\footnotesize $U$};
                                \draw[->](A)--(C);
                            }


                                \draw[->](0.15,1)--(0.85,1);
                                \draw[->](1.15,1)--(1.85,1);

                                \draw[->](2.15,1)--(2.85,1);
                                \draw[->](3.15,1)--(3.85,1);


                            \end{tikzpicture}

      	      			\end{center}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}\justifying
          \item<1-> How do we train this network ?\\
           \onslide<2-> {(\textbf{Ans}: using backpropagation)}
          \item<3-> Let us understand this with a concrete example
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      	      	      	\vspace{1.5cm}
      	      	      	\begin{center}
                            \include{modules/Module3/tikz_images/alpha}

      	      	      	\end{center}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}\justifying
          \item<1-> Suppose we consider our task of auto-completion
                    (predicting the next character)
          \item<2-> For simplicity we assume that there are only 4 characters
                    in our vocabulary (d,e,p, $<$stop$>$)
          \item<3-> At each timestep we want to predict one of these 4 characters
          \item<4-> What is a suitable output function for this task ?
            \onslide<5-> \textbf{(softmax)}
          \item<6-> What is a suitable loss function for this task ?
            \onslide<7-> \textbf{(cross entropy)}
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}


\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
                    \begin{center}
                    \include{modules/Module3/tikz_images/pre_true}
                    \end{center}
      \end{overlayarea}


		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}\justifying
				\item<1-> Suppose we initialize $U,V,W$ randomly and the network predicts
				the probabilities as shown
                \item<2-> And the true probabilities are as shown
				\item<3-> We need to answer two questions
				\item<4-> What is the total loss made by the model ?
				\item<5-> How do we backpropagate this loss and update the
				parameters ($\theta=\{U,V,W,b,c\}$) of the network ?
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
					\begin{center}
						\include{modules/Module3/tikz_images/pre_true1}
					\end{center}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}\justifying
          \item<1-> The total loss is simply the sum of the loss over all time-steps
            \begin{align*}
              \onslide<2-> {\mathscr{L}(\theta) &= \sum_{t=1}^T \mathscr{L}_t(\theta)} \\
              \onslide<3-> {\mathscr{L}_t(\theta) &= -log(y_{tc})} \\
              \onslide<4-> {y_{tc} &= \text{predicted probability of true } \\
                                   & \quad \text{ character at time-step } t } \\
              \onslide<5-> {T &= \text{number of timesteps}}
            \end{align*}
          \item<6-> For backpropagation we need to compute the gradients w.r.t. $W,U,V,b,c$
          \item<7-> Let us see how to do that
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      \begin{center}
      \include{modules/Module3/tikz_images/pre_true2}
      \end{center}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}\justifying
          \item<1-> Let us consider $\frac{\partial \mathscr{L}(\theta)}{\partial V}$
                    ($V$ is a matrix so ideally we should write $\nabla_v \mathscr{L}(\theta)$)
                    \onslide<2->{\[
                      \frac{\partial \mathscr{L}(\theta)}{\partial V} =
                      \sum_{t=1}^T\frac{\partial \mathscr{L} _t(\theta)}{\partial V}
                      \]}
          \item<3-> Each term is the summation is simply the derivative of the loss w.r.t.
                    the weights in the output layer
          \item<4-> We have already seen how to do this when we studied backpropagation
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      \begin{center}
      \include{modules/Module3/tikz_images/pre_true3}
      \end{center}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}\justifying
          \item<1-> Let us consider the derivative $\frac{\partial \mathscr{L}(\theta)}{\partial W}$
          \item[]<2-> \[
                      \frac{\partial \mathscr{L}(\theta)}{\partial W} =
                      \sum_{t=1}^T\frac{\partial \mathscr{L} _t(\theta)}{\partial W}
                      \]
          \item<3-> By the chain rule of derivatives we know that
                    $\frac{\partial \mathscr{L}_t(\theta)}{\partial W}$
                    is obtained by summing gradients along all the paths
                    from $\mathscr{L}_t(\theta)$ to \footnotesize $W$
          \item<4-> What are the paths connecting $\mathscr{L}_t(\theta)$ to \footnotesize $W$ ?
          \item<5-> Let us see this by considering $\mathscr{L}_4(\theta)$
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
\begin{columns}
\column{0.5\textwidth}
\begin{overlayarea}{\textwidth}{\textheight}
\vspace{0.2cm}
\begin{center}
\include{modules/Module3/tikz_images/de_graph1}
      \end{center}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}\justifying
          \item<1-> $\mathscr{L}_4(\theta)$ depends on $s_4$
          \item<2-> $s_4$ in turn depends on $s_3$ and \footnotesize $W$
          \item<3-> $s_3$ in turn depends on $s_2$ and \footnotesize $W$
          \item<4-> $s_2$ in turn depends on $s_1$ and \footnotesize $W$
          \item<5-> $s_1$ in turn depends on $s_0$ and \footnotesize $W$ \\
                    where $s_0$ is a constant starting state.
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      \begin{center}
      \include{modules/Module3/tikz_images/de_graph}
      \end{center}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}\justifying
          \item<1-> What we have here is an ordered network
          \item<2-> In an ordered network each state variable is computed
                    one at a time in a specified order (first $s_1$, then
                    $s_2$ and so on)
          \item<3-> Now we have
                    \[
                        \frac{\partial \mathscr{L}_4(\theta)}{\partial W}
                          =
                          \frac{\partial \mathscr{L}_4(\theta)}{\partial s_4}
                          \frac{\partial s_4}{\partial W}
                    \]
          \item<4-> We have already seen how to compute
                    $\frac{\partial \mathscr{L}_4(\theta)}{\partial s_4}$
                    when we studied backprop
          \item<5-> But how do we compute $\frac{\partial s_4}{\partial W}$
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      \begin{center}
      \include{modules/Module3/tikz_images/de_graph}
      \end{center}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}\justifying
          \item<1-> Recall that
            \begin{align*}
              s_4 = \sigma(W s_3 + b)
            \end{align*}
          \item<2-> In such an ordered network, we can't compute
                    $\frac{\partial s_4}{\partial W}$ by simply treating
                    $s_3$ as a constant (because it also depends on \footnotesize $W$)
          \item<3-> In such networks the total derivative
                    $\frac{\partial s_4}{\partial W}$ has two parts
          \item<4-> \textbf{Explicit} : $\frac{\partial^+ s_4}{\partial W}$, treating all other
                    inputs as constant
          \item<5-> \textbf{Implicit} : Summing over all indirect paths from $s_4$ to \footnotesize $W$
          \item<6-> Let us see how to do this
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}


\begin{frame}
  \vspace{-1em}
  \begin{overlayarea}{\textwidth}{\textheight}
  \begin{align*}
    \frac{\partial s_4}{\partial W} &=
          \onslide<1->{        \underbrace{\frac{\partial^+ s_4}{\partial W}}_{\text{explicit}} +
                  \underbrace{\frac{\partial s_4}{\partial s_3}
                  \frac{\partial s_3}{\partial W}}_{\text{implicit}}} \\
           \onslide<2->{
              &=  \frac{\partial^+ s_4}{\partial W} +
                  \frac{\partial s_4}{\partial s_3}
                  \Big[
                  \underbrace{\frac{\partial^+ s_3}{\partial W}}_{\text{explicit}} +
                  \underbrace{\frac{\partial s_3}{\partial s_2}
                  \frac{\partial s_2}{\partial W}}_{\text{implicit}}
                  \Big] }\\
            \onslide<3->{
              &=  \frac{\partial^+ s_4}{\partial W} +
                  \frac{\partial s_4}{\partial s_3}\frac{\partial^+ s_3}{\partial W} +
                  \frac{\partial s_4}{\partial s_3}\frac{\partial s_3}{\partial s_2}
                  \Big[
                  \frac{\partial^+ s_2}{\partial W} +
                  \frac{\partial s_2}{\partial s_1}
                  \frac{\partial s_1}{\partial W}
                  \Big]} \\
            \onslide<4->{
              &=  \frac{\partial^+ s_4}{\partial W} +
                  \frac{\partial s_4}{\partial s_3}\frac{\partial^+ s_3}{\partial W} +
                  \frac{\partial s_4}{\partial s_3}\frac{\partial s_3}{\partial s_2}
                  \frac{\partial^+ s_2}{\partial W} +
                  \frac{\partial s_4}{\partial s_3}
                  \frac{\partial s_3}{\partial s_2}
                  \frac{\partial s_2}{\partial s_1}
                  \Big[
                  \frac{\partial^+ s_1}{\partial W}
                  \Big]
            }
  \end{align*}
 \onslide<5->{For simplicity we will short-circuit some of the paths}
  \begin{align*}
 \onslide<6->{  \frac{\partial s_4}{\partial W} &=
                 \frac{\partial s_4}{\partial s_4}\frac{\partial^+ s_4}{\partial W} +
                 \frac{\partial s_4}{\partial s_3}\frac{\partial^+ s_3}{\partial W} +
                 \frac{\partial s_4}{\partial s_2}\frac{\partial^+ s_2}{\partial W} +
                 \frac{\partial s_4}{\partial s_1}\frac{\partial^+ s_1}{\partial W}
                = \sum_{k=1}^{4}
                  \frac{\partial s_4}{\partial s_k}
                  \frac{\partial^+ s_k}{\partial W}}
  \end{align*}
  \end{overlayarea}
\end{frame}


\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      \begin{center}
      \include{modules/Module3/tikz_images/de_graph}
      \end{center}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}\justifying
          \item<1-> Finally we have
            \begin{align*}
              \onslide<2->{
              \frac{\partial \mathscr{L}_4(\theta)}{\partial W} &=
                            \frac{\partial \mathscr{L}_4(\theta)}{\partial s_4}
                            \frac{\partial s_4}{\partial W}
              } \\
              \onslide<3->{
              \frac{\partial s_4}{\partial W} &=
                            \sum_{k=1}^4
                            \frac{\partial s_4}{\partial s_k}
                            \frac{\partial^+ s_k}{\partial W}
              } \\
              \onslide<4->{
              \therefore \frac{\partial \mathscr{L}_t(\theta)}{\partial W} &=
                            \frac{\partial \mathscr{L}_t(\theta)}{\partial s_t}
                            \sum_{k=1}^t
                            \frac{\partial s_t}{\partial s_k}
                            \frac{\partial^+ s_k}{\partial W}
              }
            \end{align*}
          \item<5-> This algorithm is called backpropagation through
                    time (BPTT) as we backpropagate over all previous
                    time steps
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}
