\begin{frame}
	\myheading{Module 11.5 : Image Classification continued (GoogLeNet and ResNet)}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\input{modules/Module5/tikz_images/inception.tex}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\onslide<1->{
				\begin{itemize}
					\justifying
					\item<1-> Consider the output at a certain layer of a convolutional neural network
					\item<2-> After this layer we could apply a maxpooling layer 
					\item<3-> Or a $1\times1$ convolution
					\item<4-> Or a $3\times3$ convolution
					\item<5-> Or a $5\times5$ convolution
					\item<6-> \textbf{Question:} Why choose between these options (convolution, maxpooling, filter sizes)?
					\item<7-> \textbf{Idea:} Why not apply all of them at the same time and then concatenate the feature maps?
					
				\end{itemize}
			}
		\end{overlayarea}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\input{modules/Module5/tikz_images/inception_1.tex}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}
				\justifying
				\item<1-> Well this naive idea could result in a large number of computations
				\item<2-> If $P=0$ \& $S=1$ then convolving a $W \times H \times D$ input with a 
				$F \times F \times D$ filter results in a $(W - F + 1)(H - F + 1)$ sized output
				\item<3-> Each element of the output requires $O(F \times F \times D)$ computations
				\item<4-> Can we reduce the number of computations?
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\input{modules/Module5/tikz_images/d11_filter.tex}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}
				\justifying
				\item<1-> Yes, by using $1 \times 1$ convolutions
				\item<2-> Huh?? What does a $1 \times 1$ convolution do ?
				\item<3-> It aggregates along the depth
				\item<4-> So convolving a $D \times W \times H$ input with $D_1$ $1 \times 1\ (D_1 < D)$ filters will result
				in a $D_1 \times W \times H$ output ($S=1, P=0$)
				\item<5-> If $D_1<D$ then this effectively reduces the dimension of the input and hence the computations
				\item<6-> Specifically instead of O($F\times F\times D$) we will need O($F\times F\times D_1$) computations
				\item<7-> We could then apply subsequent $3\times 3$, $5\times 5$ filter on this reduced output  
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\input{modules/Module5/tikz_images/ince.tex}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}
				\justifying
				\item<1-> But we might want to use different dimensionality reductions before the $3 \times 3$ and $5 \times 5$ filters
				\item<2-> So we can use $D_1$ and $D_2$ $1\times 1$ filters before the $3 \times 3$ and $5 \times 5$ filters respectively
				\item<3-> We can then add the maxpooling layer followed by dimensionality reduction
				\item<4-> And a new set of $1\times 1$ convolutions
				\item<5-> And finally we concatenate all these layers
				\item<6-> This is called the \textbf{Inception module}
				\item<7-> We will now see \textbf{GoogLeNet} which contains many such inception modules
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
	\input{modules/Module5/tikz_images/sid.tex}
	\begin{overlayarea}{\textwidth}{\textheight}
		\vspace{-1cm}
		
		\only<6>{\begin{center}
			\input{modules/Module5/tikz_images/ince_3a.tex}
			\end{center}}
		\only<7>{\begin{center}
			\input{modules/Module5/tikz_images/ince_3b.tex}
			\end{center}}
		\only<9>{\begin{center}
			\input{modules/Module5/tikz_images/ince_4a.tex}
			\end{center}}
		\only<10>{\begin{center}
			\input{modules/Module5/tikz_images/ince_4b.tex}
			\end{center}}
		\only<11>{\begin{center}
			\input{modules/Module5/tikz_images/ince_4c.tex}
			\end{center}}
		\only<12>{\begin{center}
			\input{modules/Module5/tikz_images/ince_4d.tex}
			\end{center}}
		\only<13>{\begin{center}
			\input{modules/Module5/tikz_images/ince_4e.tex}
			\end{center}}
		\only<15>{\begin{center}
			\input{modules/Module5/tikz_images/ince_5a.tex}
			\end{center}}
		\only<16>{\begin{center}
			\input{modules/Module5/tikz_images/ince_5b.tex}
			\end{center}}
	\end{overlayarea}
	
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\input{modules/Module5/tikz_images/ince_2.tex}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			
			\only<1-7>{
				\begin{itemize}
					\justifying
					\item<1-> $\textbf{Important Trick:}$ Got rid of the fully connected layer
					\item<2-> Notice that output of the last layer is $7 \times 7 \times 1024$ dimensional
					\item<3-> What if we were to add a fully connected layer with $1000$ nodes (for $1000$ classes)
					on top of this
					\item<4-> We would have $7 \times 7 \times 1024 \times 1000\ = 49M\ parameters$
					\item<5-> Instead they use an average pooling of size $7 \times 7$ on each of the $1024$ feature maps 
					\item<6->This results in a $1024$ dimensional output 
					\item<7-> Significantly reduces the number of parameters
				\end{itemize}}
			\only<8-9>{
				\begin{itemize}
					\justifying
					\item<8-> $12\times$ less parameters than AlexNet
					\item<9-> $2\times$ more computations
				\end{itemize}
			}
		\end{overlayarea}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
	\begin{block}{}
		\begin{itemize}
			\item GoogLeNet
			\item \color{red}{ResNet}
		\end{itemize}
	\end{block}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% \begin{frame}
% \begin{columns}
% \column{0.5\textwidth}
% \begin{overlayarea}{\textwidth}{\textheight}
% \begin{center}
% \include{ince_3}
% \end{center}
% \end{overlayarea}
% \column{0.5\textwidth}
% \begin{overlayarea}{\textwidth}{\textheight}

% \end{overlayarea}
% \end{columns}
% \end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\only<1>{
				\vspace{0.1cm}
				\begin{tikzpicture}
					      
					\node[inner sep=0pt] (A) at (2.2,3.8)
					{\includegraphics[scale=0.6, trim={0 0 4cm 0},clip]{images/resnet_crop.png}};
				\end{tikzpicture}
				
			}
			\onslide<2->{
				\begin{tikzpicture}
					
					\node[inner sep=0pt] (A) at (5.2,3.8)
					{\includegraphics[scale = 0.6]{images/resnet_crop.png}};
					\draw[draw=orange,fill=orange!50,solid,rounded corners,opacity=0.3,line width=0.6mm] (5.1,0.4) rectangle (7.9,1.9);
					\draw[draw=orange,fill=orange!50,solid,rounded corners,opacity=0.3,line width=0.6mm] (5.1,3.8) rectangle (7.9,5.3);
				\end{tikzpicture}
				    
			}  
		\end{overlayarea}
		
		
		
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}
				\justifying
				\item<1-> Suppose we have been able to train a shallow neural network well
				\item<2-> Now suppose we construct a deeper network which has few more layers (in orange)
				\item<3-> Intuitively, if the shallow network works well then the deep network should also work well by simply learning to compute identity functions in the new layers
				\item<4-> Essentially, the solution space of a shallow neural network is a subset of the solution space of a deep neural network
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\vspace{4mm}
			\hspace{6mm}
			\onslide<1->{\begin{figure}
				\begin{center}
					\includegraphics[scale=0.25]{images/plain_nets.png}
				\end{center}
				\end{figure}
				
			}
			
		\end{overlayarea}
		
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}
				\justifying
				\item<1-> But in practice it is observed that this doesn't happen
				\item<2-> Notice that the deep layers have a higher error rate on the test set
				%\item<3-> 
				%\item<2-> Residual Networks try to fix this issue
				%\item<3-> What if we enable it to learn only a residual function of the input?
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{tikzpicture}
				\node (T) at (0,0){};
				\node [rectangle, draw=black, minimum width=2cm, minimum height=4mm, fill=red!50] (A) at ($(T) + (3,0)$) {}; 
				\node [rectangle, draw=black, minimum width=2cm, minimum height=4mm, fill=red!50] (B) at ($(A) + (0,-1)$) {}; 
				\draw [line width=1] [->] (A) -- (B);
				\node (H) at ($(B) + (0,-1)$ ) {$H(x)$};
				\node (x) at ($(A) + (0,1)$ ) {$x$};
				\node (relu1) at ($(A) + (0.4,-0.5)$ ) {\tiny{$relu$}};
				\node (relu2) at ($(B) + (0.4,-0.5)$ ) {\tiny{$relu$}};
				\draw [line width=1] [->] (x) -- (A);
				\draw [line width=1] [->] (B) -- (H);
				\onslide<3->{
					\node [rectangle, draw=black, minimum width=2cm, minimum height=4mm, fill=red!50] (C) at ($(H) + (0,-1.8)$) {}; 
					\node [rectangle, draw=black, minimum width=2cm, minimum height=4mm, fill=red!50] (D) at ($(C) + (0,-1)$) {}; 
					\draw [line width=1] [->] (C) -- (D);
					\node (F1) at ($(D) + (-0.5,-0.5)$ ) {\tiny{$F(x)$}};
					\node (x1) at ($(C) + (0,1)$ ) {$x$};
					\node (relu11) at ($(C) + (0.4,-0.5)$ ) {\tiny{$relu$}};
					\node (relu21) at ($(D) + (0.4,-0.5)$ ) {\tiny{$relu$}};
					\draw [line width=1] [->] (x1) -- (C);
					
					\node [circle,draw,line width=1] (E) at ($(D) + (0,-1)$) {};
					\draw [line width=1] [->] (D) -- (E);
					%\draw ($(x1) + (0,-0.3)$) edge[line width=1,bend right,in=90, out=90,->] ($(H1) + (0,-0.3)$){};
					\draw[->,line width=1, color=blue] ($(x1) + (0,-0.3)$) to [out=0,in=90] ($(relu21) + (1.5,1)$) to [out=-90,in=0] (E) {};
					\node (H1) at ($(E) + (0,-0.5)$ ) {\footnotesize{$H(x)=F(x)+x$}};
					\node [color=blue](I1) at ($(relu21) + (2.2,1)$) {\footnotesize{Identity}};
				}
			\end{tikzpicture}
		\end{overlayarea}
		
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}
				\item<1-> Consider any two stacked layers in a CNN
				\item<2-> The two layers are essentially learning some function of the input
				\item<3-> What if we enable it to learn only a residual function of the input?
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{tikzpicture}
				\node (T) at (0,0){};
				\node [rectangle, draw=black, minimum width=2cm, minimum height=4mm, fill=red!50] (A) at ($(T) + (3,0)$) {}; 
				\node [rectangle, draw=black, minimum width=2cm, minimum height=4mm, fill=red!50] (B) at ($(A) + (0,-1)$) {}; 
				\draw [line width=1] [->] (A) -- (B);
				\node (H) at ($(B) + (0,-1)$ ) {$H(x)$};
				\node (x) at ($(A) + (0,1)$ ) {$x$};
				\node (relu1) at ($(A) + (0.4,-0.5)$ ) {\tiny{$relu$}};
				\node (relu2) at ($(B) + (0.4,-0.5)$ ) {\tiny{$relu$}};
				\draw [line width=1] [->] (x) -- (A);
				\draw [line width=1] [->] (B) -- (H);
				
				\node [rectangle, draw=black, minimum width=2cm, minimum height=4mm, fill=red!50] (C) at ($(H) + (0,-1.8)$) {}; 
				\node [rectangle, draw=black, minimum width=2cm, minimum height=4mm, fill=red!50] (D) at ($(C) + (0,-1)$) {}; 
				\draw [line width=1] [->] (C) -- (D);
				\node (F1) at ($(D) + (-0.5,-0.5)$ ) {\tiny{$F(x)$}};
				\node (x1) at ($(C) + (0,1)$ ) {$x$};
				\node (relu11) at ($(C) + (0.4,-0.5)$ ) {\tiny{$relu$}};
				\node (relu21) at ($(D) + (0.4,-0.5)$ ) {\tiny{$relu$}};
				\draw [line width=1] [->] (x1) -- (C);
				
				\node [circle,draw,line width=1] (E) at ($(D) + (0,-1)$) {};
				\draw [line width=1] [->] (D) -- (E);
				%\draw ($(x1) + (0,-0.3)$) edge[line width=1,bend right,in=90, out=90,->] ($(H1) + (0,-0.3)$){};
				\draw[->,line width=1, color=blue] ($(x1) + (0,-0.3)$) to [out=0,in=90] ($(relu21) + (1.5,1)$) to [out=-90,in=0] (E) {};
				\node (H1) at ($(E) + (0,-0.5)$ ) {\footnotesize{$H(x)=F(x)+x$}};
				\node [color=blue](I1) at ($(relu21) + (2.2,1)$) {\footnotesize{Identity}};
				
			\end{tikzpicture}
		\end{overlayarea}
			
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}
				\justifying
				\item<1-> Why would this help?
				\item<2-> Remember our argument that a deeper version of a shallow 
				network would do just fine by learning identity transformations in the new layers
				\item<3-> This identity connection from the input allows a ResNet to retain a copy of the input
				\item<4-> Using this idea they were able to train really deep networks
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}
		
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%			
			
\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\vspace{0.3cm}
			\begin{figure}
				\begin{center}
					\includegraphics[scale=0.29]{images/resNet_layers.png}
					\caption*{ResNet, \textcolor{red} {152 layers}}
				\end{center}
			\end{figure}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{block}{$1^{st}$ place in all five main tracks}
				\begin{itemize}
					\justifying
					\item<1-> \textbf{ImageNet Classification:}\hspace{-2mm} ``Ultra-deep'' \textcolor{red}{152-layer} nets
					\item<2-> \textbf{ImageNet Detection:} \textcolor{red}{$16$\%} better than the 2nd best system
					\item<3-> \textbf{ImageNet Localization:} \textcolor{red}{$27$\%} better than the 2nd best system
					\item<4-> \textbf{COCO Detection:} \textcolor{red}{$11$\%} better than the 2nd best system
					\item<5-> \textbf{COCO Segmentation:} \textcolor{red}{$12$\%} better than the 2nd best system
				\end{itemize}
			\end{block}
		\end{overlayarea}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\vspace{0.3cm}
			\begin{figure}
				\begin{center}
					\includegraphics[scale=0.29]{images/resNet_layers.png}
					\caption*{ResNet, \textcolor{red} {152 layers}}
				\end{center}
			\end{figure}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{block}{Bag of tricks}
				\begin{itemize}
					\justifying
					\item<1-> Batch Normalizaton after every CONV layer
					\item<2-> Xavier/2 initialization from [He et al]
					\item<3-> SGD + Momentum(0.9)
					\item<4-> Learning rate:0.1, divided by 10 when validation error plateaus
					\item<5-> Mini-batch size 256
					\item<6-> Weight decay of 1e-5
					\item<7-> No dropout used
				\end{itemize}
			\end{block}
		\end{overlayarea}
	\end{columns}
\end{frame}

