\begin{frame}
	\myheading{Module 22.2: Masked Autoencoder Density Estimator (MADE)}
\end{frame}

% ====================================
\begin{frame}
	\begin{columns}
		\column{0.4\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\vspace{15mm}
				%\only<1-2>{\input{modules/Module2/tikz_images/deepauto.tex}}
				\only<3->{\input{modules/Module2/tikz_images/deepauto2.tex}}
			\end{overlayarea}
		\column{0.6\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\begin{itemize}\justifying
					\item<1-> Suppose the input $\mathbf{x} \in \{0,1\}^n$, then the output layer of an autoencoder also contains $n$ units
					\item<2-> Notice the explicit factorization of the joint distribution $p(\mathbf{x})$ also contains $n$ factors
					\begin{align*} 
						p(\mathbf{x}) = \prod_{k=1}^{n} p(x_k | \mathbf{x}_{_{<k}})
					\end{align*} 
					\item<3-> \textbf{Question:} Can we tweak an autoencoder so that its output units predict the $n$ conditional distributions instead of reconstructing the $n$ inputs?
				\end{itemize}
			\end{overlayarea}
	\end{columns}
\end{frame}
% ====================================
\begin{frame}
	\begin{columns}
		\column{0.4\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\vspace{15mm}
				\input{modules/Module2/tikz_images/deepauto2.tex}
			\end{overlayarea}
		\column{0.6\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\begin{itemize}\justifying
					\item<1-> Note that this is not straightforward because we need to make sure that the $k$-th output unit only depends on the previous $k-1$ inputs
					\item<2-> In a standard autoencoder with fully connected layers the $k$-th unit obviously depends on all the input units
					\item<3-> In simple words, there is a path from each of the input units to each of the output units
					\item<4-> We cannot allow this if we want to predict the conditional distributions $p(x_k | \mathbf{x}_{_{<k}})$ (we need to ensure that we are only seeing the \textit{given} variables $\mathbf{x}_{_{<k}}$ and nothing else)
				\end{itemize}
			\end{overlayarea}
	\end{columns}
\end{frame}
% ====================================
\begin{frame}[shrink=10]
	\begin{columns}
		\column{0.55\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\vspace{20pt}
				\scalebox{0.67}
				{
					\input{modules/Module2/tikz_images/made.tex}
				}
			\end{overlayarea}
		\column{0.45\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\begin{itemize}\justifying
					\item<1-> We could ensure this by masking some of the connections in the network to ensure that $y_k$ only depends on $\mathbf{x}_{_{<k}}$
					\item<2-> We will start by assuming some ordering on the inputs and just number them from $1$ to $n$ 
					\item<3-> Now we will \textit{randomly} assign each hidden unit a number between 1 to n-1 which indicates the number of inputs it will be connected to
					\item<4-> For example, if we assign a node the number 2 then it will be connected to the first two inputs
					\item<5-> We will do a similar assignment for all the hidden layers
				\end{itemize}
			\end{overlayarea}
	\end{columns}
\end{frame}
% ====================================
\begin{frame}[shrink=10]
	\begin{columns}
		\column{0.55\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\vspace{20pt}
				\hspace{-3mm}
				\scalebox{0.67}
				{
					\input{modules/Module2/tikz_images/made_hidden.tex}
				}
			\end{overlayarea}
		\column{0.45\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\begin{itemize}\justifying
					\item<1-> Let us see what this means
					\item<2-> For the first hidden layer this numbering is clear - it simply indicates the number of ordered inputs to which this node will be connected
					\item<3-> Let us now focus on the highlighted node in the second layer which has the number 2
					\item<4-> This node is only allowed to depend on inputs $x_1$ and $x_2$ (since it is numbered 2)
					\item<5-> This means that it should be only connected to those nodes in the previous hidden layer which have seen only $x_1$ and $x_2$
					\item<6-> In other words it should only have connections from those nodes, which have been assigned a number $\leq 2$
				\end{itemize}
			\end{overlayarea}
	\end{columns}
\end{frame}
% ====================================
\begin{frame}[shrink=10]
	\begin{columns}
		\column{0.55\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\vspace{20pt}
				\hspace{-2.5mm}
				\scalebox{0.67}
				{
					\input{modules/Module2/tikz_images/made_output.tex}
				}
			\end{overlayarea}
		\column{0.45\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\begin{itemize}\justifying
					\item<1-> Now consider the node labeled 3 in the output layer 
					\item<2-> This node is only allowed to see inputs $x_1$ and $x_2$ because it predicts $p(x_3| x_2, x_1)$ (and hence the \textit{given} variables should only be $x_1$ and $x_2$)
					\item<3-> By the same argument that we made on the previous slide, this means that it should be only connected to those nodes in the previous hidden layer which have seen only $x_1$ and $x_2$
					\item<4-> We can implement this by taking the weight matrices $W^1$, $W^2$ and $V$ and applying an appropriate mask to them so that the disallowed connections are dropped
				\end{itemize}
			\end{overlayarea}
	\end{columns}
\end{frame}
% ====================================
\begin{frame}
	% \begin{columns}
		% \column{0.4\textwidth}
		% 	\begin{overlayarea}{\textwidth}{\textheight}
		% 	\end{overlayarea}
		% \column{0.6\textwidth}
			\begin{overlayarea}{\textwidth}{0.55\textheight}
				\begin{center}
				\vspace{5mm}
				\scalebox{0.6}
				{
					\input{modules/Module2/tikz_images/made_noanim_mask.tex}
				}			
				\end{center}
			\end{overlayarea}
			\begin{overlayarea}{\textwidth}{0.45\textheight}
				\begin{itemize}\justifying
					\item<1-> For example we can apply the following mask at layer 2
					\begin{align*}
						\begin{bmatrix}
							W^2_{11} & W^2_{12} & W^2_{13} & W^2_{14} & W^2_{15} \\
							W^2_{21} & W^2_{22} & W^2_{23} & W^2_{24} & W^2_{25} \\
							W^2_{31} & W^2_{32} & W^2_{33} & W^2_{34} & W^2_{35} \\
							W^2_{41} & W^2_{42} & W^2_{43} & W^2_{44} & W^2_{45} \\
							W^2_{51} & W^2_{52} & W^2_{53} & W^2_{54} & W^2_{55}
						\end{bmatrix}
						\odot
						\begin{bmatrix}
							1 & 0 & 1 & 0 & 0 \\
							1 & 0 & 1 & 0 & 0 \\
							1 & 1 & 1 & 1 & 0 \\
							1 & 0 & 1 & 0 & 0 \\
							1 & 1 & 1 & 1 & 1
						\end{bmatrix}
					\end{align*}
				\end{itemize}
			\end{overlayarea}
	% \end{columns}
\end{frame}

% ====================================
\begin{frame}
	\begin{columns}
		\column{0.55\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\vspace{20pt}
				\scalebox{0.6}
				{
					\input{modules/Module2/tikz_images/made_noanim.tex}
				}
			\end{overlayarea}
		\column{0.45\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\begin{itemize}\justifying
					\item<1-> The objective function for this network would again be a sum of cross entropies
					\item<2-> The network can be trained using backpropagation such that the errors will only be propagated along the active (unmasked) connections (similar to what happens in dropout)
				\end{itemize}
			\end{overlayarea}
	\end{columns}
\end{frame}

% ====================================
\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\vspace{20pt}
				\scalebox{0.6}
				{
					\input{modules/Module2/tikz_images/made_sampling.tex}
				}
			\end{overlayarea}
		\column{0.5\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\begin{itemize}\justifying
					\item<1-> Similar to NADE, this model is not designed for abstraction but for generation
					\item<2-> How will you do generation in this model? \visible<3->{Using the same iterative process that we used with NADE}
					\item<4-> First sample a value of $x_1$ 
					\item<5-> Now feed this value of $x_1$ to the network and compute $y_2$
					\item<6-> Now sample $x_2$ from Bernoulli ($y_2$) and repeat the process till you generate all variables upto $x_n$
				\end{itemize}
			\end{overlayarea}
		\column{0.15\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
			\end{overlayarea}
	\end{columns}
\end{frame}

% ====================================
