\begin{frame}
	\myheading{Module 8.11 : Dropout}
\end{frame}

\begin{frame}
	\vspace{4em}
	\begin{overlayarea}{\textwidth}{\textheight}
		\begin{block}{Other forms of regularization}
			\begin{itemize}
				\item $l_2$ regularization
				\item Dataset augmentation
				\item Parameter Sharing and tying
				\item Adding Noise to the inputs
				\item Adding Noise to the outputs 
				\item Early stopping
				\item Ensemble methods
				\item \textcolor<2->{red}{Dropout}
			\end{itemize}
		\end{block}
	\end{overlayarea}
\end{frame}


\begin{frame}
	\tikzstyle{input_neuron}=[circle,draw=red!50,fill=orange!10,thick,minimum size=.2mm]
	\tikzstyle{hidden_neuron}=[circle,draw=blue!50,fill=blue!10,thick,minimum size=1mm]
	\tikzstyle{output_neuron}=[circle,draw=green!50,fill=green!20,thick,minimum size=1mm]
					
	\tikzstyle{input}=[circle,draw=black!50,fill=black!20,thick,minimum size=.2mm]
	\begin{columns}
		\column{0.5\textwidth}
		\onslide<3->{\begin{minipage}[c]{.31\textwidth}
			\input{modules/Module11/tikz_images/sample_arch1.tex}
		\end{minipage}}
		\onslide<3->{\begin{minipage}[c]{0.31\textwidth}
			\input{modules/Module11/tikz_images/sample_arch2.tex}
		\end{minipage}}
		\onslide<3->{\begin{minipage}[c]{0.31\textwidth}
			\input{modules/Module11/tikz_images/sample_arch3.tex}
		\end{minipage}}								
		\onslide<4->{\begin{minipage}{0.31\textwidth}
			\input{modules/Module11/tikz_images/network.tex}
		\end{minipage}}
		\onslide<4->{\begin{minipage}{0.31\textwidth}
			\input{modules/Module11/tikz_images/network.tex}
		\end{minipage}}
		\onslide<4->{\begin{minipage}{0.31\textwidth}
			\input{modules/Module11/tikz_images/network.tex}
		\end{minipage}}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}
				\justifying
				\item<1->  Typically model averaging(bagging ensemble) always helps
				\item<2-> Training several large neural networks for making an ensemble is prohibitively expensive
				\item<3->  Option 1: Train several neural networks having different architectures(obviously expensive)
				\item<4->  Option 2: Train multiple instances of the same network using different training samples (again expensive)
				\item<5->  Even if we manage to train with option 1 or option 2, combining several models at test time is infeasible in real time applications
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}
				
%  Lect 3.4   
\begin{frame}
	\tikzstyle{input_neuron}=[circle,draw=red!50,fill=orange!10,thick,minimum size=.2mm]
	\tikzstyle{hidden_neuron}=[circle,draw=blue!50,fill=blue!10,thick,minimum size=1mm]
	\tikzstyle{output_neuron}=[circle,draw=green!50,fill=green!20,thick,minimum size=1mm]
								
	\tikzstyle{input}=[circle,draw=black!50,fill=black!20,thick,minimum size=.2mm]
	\begin{columns}
						
		\column{0.5\textwidth}
		\onslide<1->{\begin{minipage}[c]{.31\textwidth}
			\input{modules/Module11/tikz_images/sample_arch1.tex}
		\end{minipage}}
		\onslide<1->{\begin{minipage}[c]{0.31\textwidth}
			\input{modules/Module11/tikz_images/sample_arch2.tex}
		\end{minipage}}
		\onslide<1->{\begin{minipage}[c]{0.31\textwidth}
			\input{modules/Module11/tikz_images/sample_arch3.tex}
		\end{minipage}}								
		\onslide<1->{\begin{minipage}{0.31\textwidth}
			\input{modules/Module11/tikz_images/network.tex}
		\end{minipage}}
		\onslide<1->{\begin{minipage}{0.31\textwidth}
			\input{modules/Module11/tikz_images/network.tex}
		\end{minipage}}
		\onslide<1->{\begin{minipage}{0.31\textwidth}
			\input{modules/Module11/tikz_images/network.tex}
		\end{minipage}}									
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}
				\justifying
				\item Dropout is a technique which addresses both these issues.
				\item<2-> Effectively it allows training several neural networks without any significant computational overhead.
				\item<3-> Also gives an efficient approximate way of combining exponentially many different neural networks.
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\input{modules/Module11/tikz_images/network2.tex}
	\hspace{2cm}
	\onslide<2->{
		\input{modules/Module11/tikz_images/dropout_network2.tex}
	}
	\begin{itemize}
		\item Dropout refers to dropping out units
		\item<2-> Temporarily remove a node and all its incoming/outgoing connections resulting in a thinned network
		\item<3-> Each node is retained with a fixed probability (typically $p=0.5$) for hidden nodes and $p=0.8$ for visible nodes
	\end{itemize}				
\end{frame}
			
\begin{frame}			
	\input{modules/Module11/tikz_images/network2.tex}
	\hspace{2cm}
	\input{modules/Module11/tikz_images/dropout_network2.tex}
	\begin{itemize}
		\justifying
		\item<2-> Suppose a neural network has $n$ nodes
		\item<3-> Using the dropout idea, each node can be retained or dropped
		\item<4-> For example, in the above case we drop $5$ nodes to get a thinned network
		\item<5-> Given a total of $n$ nodes, what are the total number of thinned networks that can be formed?\onslide<6-> { $2^n$}
		\item<7-> Of course, this is prohibitively large and we cannot possibly train so many networks
		\item<8-> \textbf{Trick:} (1) Share the weights across all the networks \onslide<9-> {\\(2) Sample a different network for each training instance}
		\item<10-> Let us see how?	
	\end{itemize}
							
\end{frame}

\begin{frame}			
	\input{modules/Module11/tikz_images/network2.tex}
	\hspace{2cm}
	\onslide<3-> {\input{modules/Module11/tikz_images/backprop_dropout_network2.tex}}
	\begin{itemize}
		\justifying
		\item<2-> We initialize all the parameters (weights) of the network and start training
		\item<3-> For the first training instance (or mini-batch), we apply dropout resulting in the thinned network
		\item<4-> We compute the loss and backpropagate
		\item<5-> Which parameters will we update? \onslide<6-> { Only those which are active}
	\end{itemize}
							
\end{frame}

\begin{frame}			
	\input{modules/Module11/tikz_images/network2.tex}
	\input{modules/Module11/tikz_images/f_backprop_dropout_network2.tex}
	\only<1->{\input{modules/Module11/tikz_images/dropout2_network2.tex}}	
	
	%\only<3->{\input{modules/Module11/tikz_images/f_backprop_dropout2_network2.tex}}
	
\begin{itemize}
		\justifying
		\item<1-> For the second training instance (or mini-batch), we again apply dropout resulting in a different thinned network
		\item<2-> We again compute the loss and backpropagate to the active weights
		\item<4-> If the weight was active for both the training instances then it would have received two updates by now
		\item<5-> If the weight was active for only one of the training instances then it would have received only one updates by now
		\item<6-> Each thinned network gets trained rarely (or even never) but the parameter sharing ensures that no model has untrained or poorly trained parameters
	\end{itemize}
							
\end{frame}


\begin{frame}
	\input{modules/Module11/tikz_images/network2.tex}
	\input{modules/Module11/tikz_images/dropout_train.tex}
	\onslide<4->{
		\input{modules/Module11/tikz_images/dropout_test.tex}
	}
	\begin{itemize}
		\item<2-> What happens at test time?
		\item<3-> Impossible to aggregate the outputs of $2^n$ thinned networks
		\item<4-> Instead we use the full Neural Network and scale the output of each node by the fraction of times it was on during training
	\end{itemize}
						
\end{frame}
				
% \begin{frame}
% 	\hspace{1.8cm}
% 	\input{modules/Module11/tikz_images/backprop_network2.tex}
% 	\hspace{2cm}
% 	\input{modules/Module11/tikz_images/backprop_dropout_network2.tex}				
% 	\begin{itemize}
% 		\justifying
% 		\item<2-> How do you do backpropagation in such a noisy network which changes for each training instance (or batch)
% 		\item<3-> Simple: we only backpropagate over the paths which are active and only update those weights which are active in the current thinned network
% 	\end{itemize}
% \end{frame}
					
\begin{frame}
	\begin{columns}									
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\vspace{1cm}
			\hspace{1cm}
			\input{modules/Module11/tikz_images/f_backprop_dropout_network2.tex}
		\end{overlayarea}									
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}
				\justifying
				\item Dropout essentially applies a masking noise to the hidden units
				\item<2-> Prevents hidden units from co-adapting
				\item<3-> Essentially a hidden unit cannot rely too much on other units as they may get dropped out any time
				\item<4-> Each hidden unit has to learn to be more robust to these random dropouts
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}
						
\begin{frame}
	\begin{columns}
										
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\vspace{1cm}
			\hspace{1cm}
			\input{modules/Module11/tikz_images/h_network2.tex}
		\end{overlayarea}
										
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}\justifying
				\item<2-> Here is an example of how dropout helps in ensuring redundancy and robustness
				\item<3-> Suppose $h_i$ learns to detect a face by firing on detecting a nose
				\item<4-> Dropping $h_i$ then corresponds to erasing the information that  a nose exists
				\item<5-> The model should then learn another $h_i$ which redundantly encodes the presence of a nose
				\item<6-> Or the model should learn to detect the face using other features
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\vspace{4em}
	\begin{overlayarea}{\textwidth}{\textheight}
		\begin{block}{Recap}
			\begin{itemize}
				\item $l_2$ regularization
				\item Dataset augmentation
				\item Parameter Sharing and tying
				\item Adding Noise to the inputs
				\item Adding Noise to the outputs 
				\item Early stopping
				\item Ensemble methods
				\item Dropout
			\end{itemize}
		\end{block}
	\end{overlayarea}
\end{frame}
				
				
		%%%%%%%%%%%%The End%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%5555
				
				
