\begin{frame}
	\myheading{Module 5.7 : Tips for Adjusting learning Rate and Momentum}
\end{frame}

\begin{frame}
	\fontsize{16pt}{7.2}\selectfont
	\textit{Before moving on to advanced optimization algorithms let us revisit the problem of learning rate in gradient descent}
\end{frame}

\begin{frame}
	\begin{columns}
		\column{0.57\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}\justifying
				\item One could argue that we could have solved the problem of navigating gentle slopes by setting the learning rate high (i.e., blow up the small gradient by multiplying it with a large $\eta$) 
				      \item<2-> Let us see what happens if we set the learning rate to 10
				      \item<18-> On the regions which have a steep slope, the already large gradient blows up further
				      \item<19-> It would be good to have a learning rate which could adjust to the gradient ... \onslide<20->{we will see a few such algorithms soon}
			\end{itemize}
		\end{overlayarea}
		
		\column{0.43\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{figure}
				\foreach \n in {0,...,20} {%
					\pgfmathtruncatemacro\result{int(round(\n + 2))}
					\begin{tikzpicture}
						\sbox0{\includegraphics[scale=0.38]{images/module7/sgd0.3/2d_path\result.png}}% get width and height
						\only<\n>{\node[above right,inner sep=0pt] at (0,0)  {\usebox{0}}};
						\only<\n>{\node[red,font=\small](w) at (0.5\wd0,0.23\ht0) {w}};
						\only<\n>{\node[red,font=\small](b) at (0,0.67\ht0) {b}};
						\only<\n>{\draw[->, line width=0.2mm](w)--(0.6\wd0,0.23\ht0)};
						\only<\n>{\draw[->, line width=0.2mm](b)--(0,0.8\ht0)};
					\end{tikzpicture}
				}
			\end{figure}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\begin{overlayarea}{\textwidth}{\textheight}
		\onslide<1->{
			\begin{block}{Tips for initial learning rate ?}
				\onslide<2->{
					\begin{itemize}\justifying
						\item<2-> Tune learning rate \onslide<2->{[Try different values on a log scale: 0.0001, 0.001, 0.01, 0.1. 1.0]}
						\item<3-> Run a few epochs with each of these and figure out a learning rate which works best
						\item<4-> Now do a finer search around this value  [for example, if the best learning rate was 0.1 then now try some values around it: 0.05, 0.2, 0.3]
						\item<5-> Disclaimer: these are just heuristics ... no clear winner strategy
					\end{itemize}
				}
			\end{block}
		}
	\end{overlayarea}
\end{frame}

\begin{frame}
	\begin{overlayarea}{\textwidth}{\textheight}
		\only<1->{
			\begin{block}{Tips for annealing learning rate}
				\onslide<2->{
					\begin{itemize}\justifying
						\item<2-> \textbf{Step Decay:}
						\begin{itemize}\justifying
							\item<3-> Halve the learning rate after every 5 epochs or
							\item<4-> Halve the learning rate after an epoch if the validation error is more than what it was at the end of the previous epoch 
						\end{itemize}
						\item<5-> \textbf{Exponential Decay:} $\eta = \eta_{0}^{-kt}$ where $\eta_{0}$ and $k$ are hyperparameters and $t$ is the step number
						\item<6-> \textbf{1/t Decay:} $\eta = \frac{\eta_{0}}{1 + kt}$ where $\eta_{0}$ and $k$ are hyperparameters and $t$ is the step number
					\end{itemize}
				}
			\end{block}
		}
	\end{overlayarea}
\end{frame}


\begin{frame}
	\begin{overlayarea}{\textwidth}{\textheight}
		\begin{block}{Tips for momentum}
			\begin{itemize}\justifying
				\item The following schedule was suggested by Sutskever \textit{et. al.}, 2013
				      \begin{align*}
				      	\gamma_t = min (1 - 2^{-1 - log_2(\lfloor {t/250} \rfloor + 1)}, \gamma_{max})      \\
				      	\intertext{where, $\gamma_{max}$ was chosen from \{0.999, 0.995, 0.99, 0.9, 0\}} 
				      \end{align*}
			\end{itemize}
		\end{block}
	\end{overlayarea}
\end{frame}
