\begin{frame}
	\myheading{Module 14.2: Recurrent Neural Networks}
\end{frame}

\begin{frame}
  How do we model such tasks involving sequences ?
\end{frame}

\begin{frame}
\begin{block}{Wishlist}
        \begin{itemize} \justifying
          \item<1-> Account for dependence between inputs
          \item<2-> Account for variable number of inputs
          \item<3-> Make sure that the function executed at each
                    time step is the same
          \item<4-> We will focus on each of these to arrive at a model for
                    dealing with sequences
        \end{itemize}\end{block}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      	      	      	      	\vspace{1.5cm}
      	      	      	      	\begin{center}
      	      	      	      		\begin{tikzpicture}

      	      	      	      		\onslide<2->{
      	      	      	      			\node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (0,1){};
      	      	      	      			\node at (0.4,1) {$s_1$};
					\node at (0.2,1.8) {\footnotesize $V$};
					\node at (0.2,0.2) {\footnotesize $U$};
      	      	      	      			\node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (0,-0.5){};
      	      	      	      			\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (0,-1){\tiny $x_1$};
      	      	      	      			\draw[->](Aemb)--(A);
      	      	      	      		}
      	      	      	      		\onslide<2->{
      	      	      	      			\node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (0,2.8){};
      	      	      	      			\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (0,3.3){\tiny $y_1$};

      	      	      	      			\draw[->](A)--(C);
      	      	      	      		}


      	      	      	      		\onslide<2->{
      	      	      	      			\node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (1,1){};
      	      	      	      			\node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (1,-0.5){};
      	      	      	      			\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (1,-1){\tiny $x_2$};
      	      	      	      			\draw[->](Aemb)--(A);
      	      	      	      			\node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (1,2.8){};
      	      	      	      			\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (1,3.3){\tiny $y_2$};
     	      	      	      				\node at (1.4,1) {$s_2$};
      	      	      	      				\node at (1.2,1.8) {\footnotesize $V$};
      	      	      	      				\node at (1.2,0.2) {\footnotesize $U$};
      	      	      	      			\draw[->](A)--(C);
      	      	      	      		}


      	      	      	      		\end{tikzpicture}

      	      	      	      	\end{center}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize} \justifying
          \item<1-> What is the function being executed at each time
                    step ?
                    \begin{align*}
                      \onslide<2-> {s_i &= \sigma(U x_i + b)} \\
                      \onslide<3-> {y_i &= \mathcal{O}(V s_i + c)} \\
                      \onslide<4-> { i &= \text{timestep}}
                    \end{align*}
          \item<5-> Since we want the same function to be executed at
                    each timestep we should share the same network
                    (i.e., same parameters at each timestep)
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      			\vspace{1.5cm}
      			\begin{center}
      				\begin{tikzpicture}
      				\onslide<1->{
      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (0,1){};
      					\node at (0.4,1) {$s_1$};
      					\node at (0.2,1.8) {\footnotesize $V$};
      					\node at (0.2,0.2) {\footnotesize $U$};
      					\node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (0,-0.5){};
      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (0,-1){\tiny $x_1$};
      					\draw[->](Aemb)--(A);
      				}
      				\onslide<1->{
      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (0,2.8){};
      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (0,3.3){\tiny $y_1$};

      					\draw[->](A)--(C);
      				}


      				\onslide<1->{
      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (1,1){};
      					\node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (1,-0.5){};
      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (1,-1){\tiny $x_2$};
      					\draw[->](Aemb)--(A);
      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (1,2.8){};
      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (1,3.3){\tiny $y_2$};
      					\node at (1.4,1) {$s_2$};
      					\node at (1.2,1.8) {\footnotesize $V$};
      					\node at (1.2,0.2) {\footnotesize $U$};
      					\draw[->](A)--(C);
      				}
      				\onslide<4->{
      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (2,1){};
      					\node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (2,-0.5){};
      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (2,-1){\tiny $x_3$};
      					\draw[->](Aemb)--(A);
      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (2,2.8){};
      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (2,3.3){\tiny $y_3$};
      					\node at (2.4,1) {$s_3$};
      					\node at (2.2,1.8) {\footnotesize $V$};
      					\node at (2.2,0.2) {\footnotesize $U$};
      					\draw[->](A)--(C);
      				}
      				\onslide<5->{
      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (3,1){};
      					\node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (3,-0.5){};
      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (3,-1){\tiny $x_4$};
      					\draw[->](Aemb)--(A);
      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (3,2.8){};
      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (3,3.3){\tiny $y_4$};
      					\node at (3.4,1) {$s_4$};
      					\node at (3.2,1.8) {\footnotesize $V$};
      					\node at (3.2,0.2) {\footnotesize $U$};
      					\draw[->](A)--(C);
      				}
      				\onslide<6->{			\node (A) at (4,1){\dots};
      				}
      				\onslide<7->{
      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (5,1){};
      					\node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (5,-0.5){};
      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (5,-1){\tiny $x_n$};
      					\draw[->](Aemb)--(A);
      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (5,2.8){};
      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (5,3.3){\tiny $y_n$};
      					\node at (5.4,1) {$s_n$};
      					\node at (5.2,1.8) {\footnotesize $V$};
      					\node at (5.2,0.2) {\footnotesize $U$};
      					\draw[->](A)--(C);
      				}
      				\end{tikzpicture}

      			\end{center}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}\justifying
          \item<1-> This parameter sharing also ensures that the network
                    becomes agnostic to the length (size) of the input
          \item<2-> Since we are simply going to compute the same function
                    (with same parameters) at each timestep, the number of
                    timesteps doesn't matter
          \item<3-> We just create multiple copies of the network and execute
                    them at each timestep
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}

      \include{modules/Module2/tikz_images/mult}

      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}\justifying
          \item<1-> How do we account for dependence between inputs ?
          \item<2-> Let us first see an infeasible way of doing this
          \item<3-> At each timestep we will feed all the previous inputs to the network
          \item<6-> Is this okay ?
          \item<7-> No, it violates the other two items on our wishlist
          \item<8-> How ? \onslide<9->{Let us see}
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \include{modules/Module2/tikz_images/mult1}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}\justifying
          \item<1-> First, the function being computed at each time-step now is different
            \begin{align*}
              \onslide<2-> {y_1 &= f_1(x_1)} \\
              \onslide<3-> {y_2 &= f_2(x_1, x_2)} \\
              \onslide<4-> {y_3 &= f_3(x_1, x_2, x_3)}
            \end{align*}
          \item<5-> The network is now sensitive to the length of the sequence
          \item<6-> For example a sequence of length 10 will require $f_1,\ldots,f_{10}$
                    whereas a sequence of length 100 will require $f_1,\ldots,f_{100}$
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      	      			\vspace{1.5cm}
      	      			\begin{center}
      	      				\begin{tikzpicture}
      	      				\onslide<1->{
      	      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (0,1){};
      	      					\node at (0.5,1.2) {\footnotesize $W$};
      	      					\node at (0.2,1.8) {\footnotesize $V$};
      	      					\node at (0.2,0.2) {\footnotesize $U$};
      	      					\node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (0,-0.5){};
      	      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (0,-1){\tiny $x_1$};
      	      					\draw[->](Aemb)--(A);
      	      				}
      	      				\onslide<1->{
      	      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (0,2.8){};
      	      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (0,3.3){\tiny $y_1$};

      	      					\draw[->](A)--(C);
      	      				}


      	      				\onslide<1->{
      	      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (1,1){};
      	      					\node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (1,-0.5){};
      	      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (1,-1){\tiny $x_2$};
      	      					\draw[->](Aemb)--(A);
      	      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (1,2.8){};
      	      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (1,3.3){\tiny $y_2$};
      	      					\node at (1.5,1.2) {\footnotesize $W$};
      	      					\node at (1.2,1.8) {\footnotesize $V$};
      	      					\node at (1.2,0.2) {\footnotesize $U$};
      	      					\draw[->](A)--(C);
      	      				}
      	      				\onslide<1->{
      	      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (2,1){};
      	      					\node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (2,-0.5){};
      	      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (2,-1){\tiny $x_3$};
      	      					\draw[->](Aemb)--(A);
      	      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (2,2.8){};
      	      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (2,3.3){\tiny $y_3$};
      	      					\node at (2.5,1.2) {\footnotesize $W$};
      	      					\node at (2.2,1.8) {\footnotesize $V$};
      	      					\node at (2.2,0.2) {\footnotesize $U$};
      	      					\draw[->](A)--(C);
      	      				}
      	      				\onslide<1->{
      	      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (3,1){};
      	      					\node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (3,-0.5){};
      	      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (3,-1){\tiny $x_4$};
      	      					\draw[->](Aemb)--(A);
      	      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (3,2.8){};
      	      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (3,3.3){\tiny $y_4$};
      	      					\node at (3.5,1.2) {\footnotesize $W$};
      	      					\node at (3.2,1.8) {\footnotesize $V$};
      	      					\node at (3.2,0.2) {\footnotesize $U$};
      	      					\draw[->](A)--(C);
      	      				}
      	      				\onslide<1->{			\node (A) at (4,1){\dots};
      	      				}
      	      				\onslide<1->{
      	      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (5,1){};
      	      					\node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (5,-0.5){};
      	      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (5,-1){\tiny $x_n$};
      	      					\draw[->](Aemb)--(A);
      	      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (5,2.8){};
      	      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (5,3.3){\tiny $y_n$};
      	      					\node at (5.4,1) {$s_n$};
      	      					\node at (4.5,1.2) {\footnotesize $W$};
      	      					\node at (5.2,1.8) {\footnotesize $V$};
      	      					\node at (5.2,0.2) {\footnotesize $U$};
      	      					\draw[->](A)--(C);
      	      					\draw[->](0.15,1)--(0.85,1);
								\draw[->](1.15,1)--(1.85,1);
								\draw[->](2.15,1)--(2.85,1);
								\draw[->](3.15,1)--(3.75,1);				\draw[->](4.35,1)--(4.85,1);
      	      				}
      	      				\end{tikzpicture}

      	      			\end{center}
      	      		\end{overlayarea}


    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}\justifying
          \item<1-> The solution is to add a recurrent connection in the network,
            \begin{align*}
              \onslide<2-> {s_i &= \sigma(U x_i + W s_{i-1} + b)} \\
              \onslide<3-> {y_i &= \mathcal{O}(V s_i + c)} \\
              \onslide<4-> {& or} \\
              \onslide<5-> {y_i &= f(x_i, s_{i-1}, W, U, V, b, c)}
            \end{align*}
          \item<6-> $s_i$ is the state of the network at timestep $i$
          \item<7-> The parameters are $W,U,V, c, b$ which are shared across timesteps
          \item<8-> The same network (and parameters) can be used to compute
                    $y_1,y_2,\ldots,y_{10}$ or $y_{100}$
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}


\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      	      	      			\vspace{1.5cm}
      	      	      			\begin{center}
      	      	      				\begin{tikzpicture}
      	      	      				\onslide<1->{
      	      	      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=blue!50, draw=black] (A) at (0,1){};
      	      	      					\node at (-0.4,1) {$s_i$};
      	      	      					\node at (0.2,1.8) {\footnotesize $V$};
      	      	      					\node at (0.2,0.2) {\footnotesize $U$};
      	      	      					\node [rectangle, minimum height=6mm, minimum width=3mm, fill=orange!50, draw=black] (Aemb) at (0,-0.5){};
      	      	      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (0,-1){\tiny $x_i$};
      	      	      					\draw[->](Aemb)--(A);
\path  (A)   edge[out=300,in=420,looseness=8,->] node[right]  {\footnotesize $W$} (A);
      	      	      				}
      	      	      				\onslide<1->{
      	      	      					\node [rectangle, minimum height=8mm, minimum width=3mm, fill=green!50, draw=black] (C) at (0,2.8){};
      	      	      					\node [rectangle, minimum height=3mm, minimum width=3mm,text width=1cm, align=center] (w1) at (0,3.3){\tiny $y_i$};

      	      	      					\draw[->](A)--(C);
      	      	      				}

      	      	      				\end{tikzpicture}

      	      	      			\end{center}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}\justifying
          \item This can be represented more compactly
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}



\begin{frame}
    \begin{columns}
        \column{0.5\textwidth}
        \begin{overlayarea}{\textwidth}{\textheight}

            \begin{center}

            \include{modules/Module2/tikz_images/deep}

            \end{center}

        \end{overlayarea}


        \column{0.5\textwidth}
        \begin{overlayarea}{\textwidth}{\textheight}
            \begin{itemize}\justifying
                \item<1-> Let us revisit the sequence learning problems that we saw earlier
                \item<2-> We now have recurrent connections between time steps which account for dependence between inputs
            \end{itemize}
        \end{overlayarea}
    \end{columns}
\end{frame}
