\begin{frame}
	\myheading{Module 22.1: Neural Autoregressive Density Estimator (NADE)}
\end{frame}


\begin{frame}
	\begin{columns}
		\column{0.35\textwidth}
		\begin{overlayarea}{\textwidth}{0.4\textheight}
			\begin{center}
				\scalebox{0.55}{\input{modules/Module1/tikz_images/rbm.tex}}
			\end{center}
		\end{overlayarea}

		\begin{overlayarea}{\textwidth}{0.6\textheight}
			\begin{center}
				\scalebox{0.5}{\input{modules/Module1/tikz_images/vae.tex}}
			\end{center}
		\end{overlayarea}

		\column{0.65\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}[<+->]\justifying
				\item So far we have seen a few latent variable generation models such as RBMs and VAEs
				\item Latent variable models make certain independence assumptions which reduces the number of factors and in turn the number of parameters in the model
				\item For example, in RBMs we assumed that the visible variables were independent given the hidden variables which allowed us to do Block Gibbs Sampling
				\item Similarly in VAEs we assumed $P(\textbf{x}|z) = \mathcal{N}(0, I)$ which effectively means that given the latent variables, the $\mathbf{x}$'s are independent of each other (Since $ \Sigma $ = I)
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}


\begin{frame}
	\begin{columns}
		\column{0.4\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\vspace{30mm}	
			\visible<4->{\input{modules/Module1/tikz_images/dgm.tex}}
			%LHS: on B4 show a fully dependent directed graphical model 
		\end{overlayarea}

		\column{0.6\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}\justifying
				\item<1-> We will now look at Autoregressive (AR) Models which do not contain any latent variables
				\item<2-> The aim of course is to learn a joint distribution over $\mathbf{x}$ 
				\item<3-> As usual, for ease of illustration we will assume $ \mathbf{x} \in {\{0,1\}}^n $
				\item<4-> AR models do not make any independence assumption but use the default factorization of $p(\mathbf{x})$ given by the chain rule
					$ p(\mathbf{x}) = {\displaystyle \prod_{i=1}^{n}} p(x_i | \mathbf{x}_{<k})  $
%			    \onslide<5->{(they were able to do \textit{generation})}
				\item<5-> The above factorization contains $n$ factors and some of these factors contain many parameters ( $O(2^n)$ in total )
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}


\begin{frame}
	\begin{columns}
		\column{0.4\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\input{modules/Module1/tikz_images/dgm.tex}
			%LHS: Same as above
		\end{overlayarea}
		\column{0.6\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}[<+->]\justifying
				\item Obviously, it is infeasible to learn such an exponential number of parameters
				\item AR models work around this by using a neural network to parameterize these factors and then learn the parameters of this neural network
				\item What does this mean? Let us see!
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}


\begin{frame}
	\begin{columns}
		\column{0.4\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\vspace*{10mm}
			\begin{center}
				\hspace{-10mm}
				\scalebox{0.85}{\input{modules/Module1/tikz_images/nade0.tex}}
			\end{center}
			%LHS: On B4 show output layer with n nodes and on top of each node write down the corresponding distribution P(x_1) P(x_2|x_1), P(x_3 | x_2, x_1) and so on
		\end{overlayarea}
		\column{0.6\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}\justifying
				 \item<1-> At the output layer we want to predict $n$ conditional probability distributions (each corresponding to one of the factors in our joint distribution)
				\item<2-> At the input layer we are given the $n$ input variables
				\item<3-> Now the catch is that the $n^{th}$ output should only be connected to the previous $n$-$1$ inputs 
				\item<4-> In particular, when we are computing $ p({x}_3 | {x}_2, {x}_1) $ the only inputs that we should consider are ${x}_1$, ${x}_2$ because these are the only variables \textit{given} to us while computing the conditional
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\begin{columns}
		\column{0.4\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\vspace*{10mm}
			\begin{center}
				\hspace{-10mm}
				\only<1-2>{\scalebox{0.85}{\input{modules/Module1/tikz_images/nade.tex}}}
				\only<3->{\scalebox{0.85}{\input{modules/Module1/tikz_images/nade_highlighted.tex}}}
			\end{center}

			% \scalebox{0.85}{\input{modules/Module1/tikz_images/nade_nn2.tex}}
			%LHS: Same figure as above On bullet 3 show a W \in R^3x3, x \in {0,1}3, highlight W.,<2, x_<2 
		\end{overlayarea}
		\column{0.6\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}\justifying
				\item<1-> The Neural Autoregressive Density Estimator (NADE) proposes a simple solution for this
				\item<2-> First, for every output unit, we compute a hidden representation using only the relevant input units
				\item<3-> For example, for the $k^{th}$ output unit, the hidden representation will be computed using:
				\[{ h_k = \sigma(W_{.,\textless k} \textbf{x}_{ \textless k} + b) }^{}\]
				where $ h_k \in R^d, W \in R^{d\times n} ,  W_{.,<k} $ are the first $k$ columns of $W$
				\item<4-> We now compute the output $ p(x_k|\textbf{x}_{1}^{k-1}) $ as:
				\[ y_k = p(x_k|\textbf{x}_{1}^{k-1}) = \sigma(V_k h_k + c_k) \]
			\end{itemize}
			\vspace{10mm}
			% \only<3->{\footnotetext{$^{**}$(see equations at 21:51 of this video\\ \url{https://www.youtube.com/watch?v=R8fx2b8Asg0})}}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\begin{columns}
		\column{0.4\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\vspace*{10mm}
			\begin{center}
				\hspace{-10mm}
				\scalebox{0.85}{\input{modules/Module1/tikz_images/nade_highlighted.tex}}
			\end{center}
			% \scalebox{0.85}{\input{modules/Module1/tikz_images/nade_nn2.tex}}
			% LHS: Same as above
		\end{overlayarea}
		\column{0.6\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}\justifying
				\item<1-> Let us look at the equations carefully
				\[ h_k = \sigma(W_{.,<k} \textbf{x}_{<k} + b) \]
				\[ y_k = p(x_k|\textbf{x}_{1}^{k-1}) = \sigma(V_k h_k + c_k) \]
				\item<2-> How many parameters does this model have ?
				\item<3-> Note that $ W \in \mathbb{R}^{d\times n} $ and $ b \in \mathbb{R}^{d \times 1} $ are shared parameters and the same $W,b$ are used for computing $ h_k $ for all the $n$ factors (of course only the relevant columns of $W$ are used for each $k$) resulting in $nd+d$ parameters
				\item<4-> In addition, we have $ V_k \in \mathbb{R}^{d\times 1} $ and $ c_k \in \mathbb{R}^{d\times 1} $ for each of the $n$ factors resulting in a total of $nd+n$ parameters
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\begin{columns}
		\column{0.4\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\vspace*{10mm}
			\begin{center}
				\hspace{-10mm}
				\only<1>{\scalebox{0.85}{\input{modules/Module1/tikz_images/nade_highlighted_h1.tex}}}
				\only<2->{\scalebox{0.85}{\input{modules/Module1/tikz_images/nade_highlighted.tex}}}
			\end{center}
			% \scalebox{0.85}{\input{modules/Module1/tikz_images/nade_nn2.tex}}
			% LHS: Same as above
		\end{overlayarea}
		\column{0.6\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}\justifying
				\item<1-> There is also an additional parameter $h_1 \in R^{d}$ (similar to the initial state in LSTMs, RNNs) 
				\item<2-> The total number of parameters in the model is thus $2nd+n+2d$ which is linear in $n$
				\item<3-> In other words, the model does not have an exponential number of parameters which is typically the case for the default factorization
				$ p(\textbf{x}) = {\displaystyle \prod_{i=1}^{n}} p(x_i | \textbf{x}_{<k}) $
				\item<4-> Why? \visible<5->{Because we are sharing the parameters across the factors}
				\item<6-> The same $W, b$ contribute to all the factors
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\begin{columns}
		\column{0.4\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\vspace*{10mm}
			\begin{center}
				\hspace{-10mm}
				\scalebox{0.85}{\input{modules/Module1/tikz_images/nade_highlighted.tex}}
			\end{center}
			% \scalebox{0.85}{\input{modules/Module1/tikz_images/nade_nn2.tex}}
			% LHS: Same as above, on B5 show the equation for the total loss function as a sum of $n$ cross entropies

		\end{overlayarea}
		\column{0.6\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}\justifying
				\item<1-> How will you train such a network? \\ \pause\textit{backpropagation: its a neural network after all}
				\item<3-> What is the loss function that you will choose? 
				\item<4-> For every output node we know the true probability distribution
				\item<5-> For example, for a given training instance, if $ X_3 =1 $ then the true probability distribution is given by $ p(\textbf{x}_3 = 1 | x_2, x_1) =1 , p(\textbf{x}_3 = 0 | x_2, x_1) = 0$ or $p=[0,1] $
				\item<6-> If the predicted distribution is $q = [0.7, 0.3]$ then we can just take the cross entropy between $p$ and $q$ as the loss function
				\item<7-> The total loss will be the sum of this cross entropy loss for all the $n$ output nodes
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\begin{columns}
		\column{0.4\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\vspace*{10mm}
			\begin{center}
				\hspace{-10mm}
				\scalebox{0.85}{\input{modules/Module1/tikz_images/nade_highlighted.tex}}
			\end{center}

			% \scalebox{0.85}{\input{modules/Module1/tikz_images/nade_nn2.tex}}
			% LHS: Same as above
		\end{overlayarea}
		\column{0.6\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}\justifying
				\item<1-> Now let's ask a couple of questions about the model (assume training is done)
				\item<2-> Can the model be used for abstraction? i.e., if we give it a test instance $\textbf{x}$, can the model give us a hidden abstract representation for $\textbf{x}$
				\item<3-> Well, you will get a sequence of hidden representations $ h_1, h_2, ..., h_n $ but these are not really the kind of abstract representations that we are interested in
				\item<4-> For example, $ h_n $ only captures the information required to reconstruct $ x_n $ given $ x_1$ to $x_{n-1} $ (compare this with an autoencoder wherein the hidden representation can reconstruct all of $ x_1, x_2 ,...,x_n) $
				\item<5-> These are not latent variable models and are, by design, not meant for abstraction
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\begin{columns}
		\column{0.4\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\vspace*{10mm}
			\begin{center}
				\hspace{-10mm}
				\only<1>{\scalebox{0.85}{\input{modules/Module1/tikz_images/nade_gen0.tex}}}
				\only<2->{\scalebox{0.85}{\input{modules/Module1/tikz_images/nade_gen1.tex}}}
			\end{center}
			% LHS: Show the generation process using an animation

		\end{overlayarea}
		\column{0.6\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}\justifying
				\item<1-> Can the model to generation? How?
				\item<2-> Well, we first compute $ p(\textbf{x}_1 = 1) $ as
				$ y_1 = \sigma(V_1h_1 + c_1) $
				\item<3-> Note that $ V_1,h_1, c_1 $ are all parameters of the model which will be learned during training
				\item<4-> We will then sample a value for $ x_1 $ from the distribution $Bernoulli ( y_1 )$
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\begin{columns}
		\column{0.4\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\vspace*{10mm}
			\begin{center}
				\hspace{-10mm}
				\only<1-3>{\scalebox{0.85}{\input{modules/Module1/tikz_images/nade_gen2.tex}}}
				\only<4->{\scalebox{0.85}{\input{modules/Module1/tikz_images/nade_gen4.tex}}}
			\end{center}
		\end{overlayarea}
		\column{0.6\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}\justifying
				 \item<1-> We will now use the sampled value of $ x_1 $ and compute $ h_2 $ as
				
				$ h_2 = \sigma(W_{_{.,<2}} \textbf{x}_{_{<2}} + b) $
				\item<2-> Using $ h_2 $ we will compute $ P(\textbf{x}_2 = 1| \mathbf{x}_1 = x_1) $ as
				$ y_2 = \sigma(V_2h_2 + c_2) $
				\item<3-> We will then sample a value for $ x_2 $ from the distribution $Bernoulli ( y_2 )$
				\item<4-> We will then continue this process till $ x_n $ generating the value of one random variable at a time
				\item<5-> If $\textbf{x}$ is an image then this is equivalent to generating the image one pixel at a time (very slow)
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}


\begin{frame}
	\begin{itemize}
		\item<1-> Of course, the model requires a lot of computations because for generating each pixel we need to compute 
			\begin{align*}
				h_k &= \sigma(W_{_{.,<k}} \textbf{x}_{_{<k}} + b)\\
				y_k &= p(x_k|\textbf{x}_{1}^{k-1}) = \sigma(V_k h_k + c_k)
			\end{align*}
		\item<2-> However notice that 
			\begin{align*}
				W_{_{.,<k+1}} \textbf{x}_{_{<k+1}} + b = W_{_{.,<k}} \textbf{x}_{_{<k}} + b + W_{_{.,k}}\mathbf{x}_{_k}
			\end{align*}
		\item<3-> Thus we can reuse some of the computations done for pixel $k$ while predicting the pixel $k+1$ (this can be done even at training time)
	\end{itemize}
\end{frame}

\begin{frame}
	\begin{block}{Things to remember about NADE}
		\begin{itemize}[<+->]
			\item Uses the explicit representation of the joint distribution $p(\mathbf{x}) = \prod\limits_{i=1}^{n} p(x_i | \mathbf{x}_{_{<k}})$
			\item Each node in the output layer corresponds to one factor in this explicit representation
			\item Reduces the number of parameters by sharing weights in the neural network
			\item Not designed for abstraction
			\item Generation is slow because the model generates one pixel (or one random variable) at a time
			\item Possible to speed up the computation by reusing some previous computations
		\end{itemize}
	\end{block}
\end{frame}
