\begin{frame}
	\myheading{Module 8.10 : Ensemble methods}
\end{frame}

\begin{frame}
	\vspace{4em}
	\begin{overlayarea}{\textwidth}{\textheight}
		\begin{block}{Other forms of regularization}
			\begin{itemize}
				\item $l_2$ regularization
				\item Dataset augmentation
				\item Parameter Sharing and tying
				\item Adding Noise to the inputs
				\item Adding Noise to the outputs 
				\item Early stopping
				\item \textcolor<2->{red}{Ensemble methods}
				\item Dropout
			\end{itemize}
		\end{block}
	\end{overlayarea}
\end{frame}

\begin{frame}
	\begin{columns}
		\column{0.4\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\justifying
			\only<1->{
				\input{modules/Module10/tikz_images/pic1}
			}
		\end{overlayarea}
		\column{0.6\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}
				\justifying
							
				\item <1-> Combine the output of different models to reduce generalization error
				      \item<2->  The models can correspond to different classifiers
				      \item<3-> It could be different instances of the same classifier trained with:
				      \begin{itemize}
				      	\justifying
				      	\item<4-> different hyperparameters
				      	\item<5-> different features
				      	\item<6-> different samples of the training data
				      \end{itemize}
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}


\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
					
			\only<1->{
						\input{modules/Module10/tikz_images/pic2}
				}
			\onslide<6->{Each model trained with a different sample of the data (sampling with replacement)}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}
				\justifying
				\item \onslide<3->  Bagging: form an ensemble using different instances of the same classifier
				\item \onslide<4-> From a given dataset, construct multiple training sets by sampling with replacement $(T_1, T_2, ... ,T_k)$
				\item \onslide<5-> Train $i^{th}$ instance of the classifier using training set $T_i$
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}
				\item <7-> The error made by the average prediction of all the models is $\frac{1}{k}\sum_{i}\varepsilon_i$
				\item <8-> The expected squared error is :
%				\vspace{-2cm}
				\scalebox{0.88}{\parbox{\linewidth}{%
					\begin{align*}
				 		%DeclareMathSizes{10}{10}{10}{10}
						\onslide<9-> {mse= & E[(\frac{1}{k}\sum_{i}\varepsilon_i)^{2}] \\} 
						\onslide<10-> {=   & \frac{1}{k^{2}}E[\sum_{i}\sum_{i=j} \varepsilon_i \varepsilon_j + \sum_{i}\sum_{i\neq j} \varepsilon_i \varepsilon_j] \\}
						\onslide<11-> {=   & \frac{1}{k^{2}}E[\sum_{i} \varepsilon_i^{2} + \sum_{i}\sum_{i\neq j} \varepsilon_i \varepsilon_j] \\}
						\onslide<12->{ =   & \frac{1}{k^{2}}(\sum_{i} E[\varepsilon_i^{2}] + \sum_{i}\sum_{i\neq j} E[\varepsilon_i \varepsilon_j]) \\}
						\onslide<13-> {=   & \frac{1}{k^{2}}(kV+k(k-1)C) \\}
						\onslide<14-> {=   & \frac{1}{k}V+\frac{k-1}{k}C  \\}
					\end{align*}
				}
			}
			% }
			\end{itemize}
		\end{overlayarea}
		\column{0.4\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}
				\justifying
				\item<1->  When would bagging work?
				\item<2-> Consider a set of $k$ LR models
				\item<3->  Suppose that each model makes an error $\varepsilon_i$ on a test example
				\item<4->  Let $\varepsilon_i$ be drawn from a zero mean multivariate normal distribution
				\item<5->  $ Variance = E[\varepsilon_i^{2}]=V$
				\item<6->  $Covariance = E[\varepsilon_i \varepsilon_j]=C$
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}


\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{equation*}
				mse=\frac{1}{k}V+\frac{k-1}{k}C
			\end{equation*}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}
				\justifying
				\item<2->  When would bagging work ?
				\item <3-> If the errors of the model are perfectly correlated then $V=C$ and $mse=V$ [bagging does not help: the mse of the ensemble is as bad as the individual models]
				\item<4->  If the errors of the model are independent or uncorrelated then $C=0$ and the mse of the ensemble reduces to $\frac{1}{k}V$
				\item <5-> On average, the ensemble will perform at least as well as its individual members
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}
