\begin{frame}
	Explanation for why we need bias correction in Adam
\end{frame}


\begin{frame}[shrink=10]
\begin{columns}
\column{0.4\textwidth}
\begin{overlayarea}{\textwidth}{\textheight}
		\begin{block}{Update rule for Adam}
					\begin{align*}
						m_t       & = \beta_{1} * m_{t-1} + (1 - \beta_{1}) * \nabla w_{t}              \\
						v_t       & = \beta_{2} * v_{t-1} + (1 - \beta_{2}) * {(\nabla w_{t})}^2        \\
						\hat{m}_t & = \frac{m_t}{1-\beta_1^t}  \\                          
						\hat{v}_t & = \frac{v_t}{1-\beta_2^t}\\
						w_{t+1}   & = w_{t} - \frac{\eta}{\sqrt{\hat{v}_{t} + \epsilon}} * \hat{m}_{t} 
					\end{align*}
		\end{block}
\end{overlayarea}
\column{0.5\textwidth}
\begin{itemize}\justifying
		\item<1-> Note that we are taking a running average of the gradients as $m_t$ \\
		\item<2-> The reason we are doing this is that we don't want to rely too much on the current gradient and instead rely on the overall behaviour of the gradients over many timesteps \\
		\item<3-> One way of looking at this is that we are interested in the expected value of the gradients and not on  a single point estimate computed at time $t$ \\
		\item<4-> However, instead of computing $E[\nabla w_t]$ we are computing $m_t$ as the exponentially moving average \\
		\item<5-> Ideally we would want $E[m_t]$ to be equal to $E[\nabla w_t]$ \\
		\item<6-> Let us see if that is the case \\
\end{itemize}
\end{columns}
\end{frame}


\begin{frame}[shrink=20]
	\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}\justifying
				\item<1-> For convenience we will denote $\nabla w_t$ as $g_t$ and $\beta_{1}$ as $\beta$
				\begin{align*}
					\onslide<2->{m_t & = \beta * m_{t-1} + (1 - \beta) * g_{t}              \\}
					\onslide<3->{m_0 & = 0  \\}
                    \onslide<4->{m_1 & = \beta m_0 + (1 - \beta)g_1 \\}
                    \onslide<5->{ & = (1 - \beta)g_1 \\}
                    \onslide<6->{m_2 & = \beta m_1 + (1 - \beta)g_2 \\}
                    \onslide<7->{& = \beta (1 - \beta)g_1 + (1 - \beta)g_2 \\}
                    \onslide<8->{m_3 & = \beta m_2 + (1 - \beta)g_3 \\}
                    \onslide<9->{& = \beta (\beta(1 - \beta)g_1 + (1 - \beta)g_2) + (1 - \beta)g_3 \\}
                    \onslide<10->{& = \beta^2(1 - \beta)g_1 + \beta(1 - \beta)g_2 + (1 - \beta)g_3 \\}
                    \onslide<11->{& = (1 - \beta) \sum_{i=1}^3 \beta^{3-i}g_i}
				\end{align*}
                \item<12-> In general,
                \begin{align*}
                    m_t & = (1-\beta)\sum_{i=1}^t \beta^{t-i} g_i
                \end{align*}
			\end{itemize}
	\end{overlayarea}
\end{frame}


\begin{frame}[shrink=20]
    \begin{columns}
        \column{0.5\textwidth}
        \begin{overlayarea}{\textwidth}{\textheight}
        \begin{itemize}
            \item<1->{So we have, $m_t=(1-\beta)\sum_{i=1}^{t}\beta^{t-i}g_i$ }
            \item<2->{Taking Expectation on both sides}
            \begin{align*}
                \visible<3->{E[m_t] &= E[(1-\beta)\sum_{i=1}^{t}\beta^{t-i}g_i]}\\
                \visible<4->{E[m_t] &= (1-\beta)E[\sum_{i=1}^{t}\beta^{t-i}g_i]}\\
                \visible<5->{E[m_t] &= (1-\beta)\sum_{i=1}^{t}E[\beta^{t-i}g_i]}\\
                \visible<6->{ &= (1-\beta)\sum_{i=1}^{t}\beta^{t-i}E[g_i]}
            \end{align*}
            \onslide<7->{\item Assumption: All $g_i$'s come from the same distribution i.e. $E[g_i] = E[g]$ $\forall i$}
        \end{itemize}
        \end{overlayarea}
        \column{0.5\textwidth}
        \begin{overlayarea}{\textwidth}{\textheight}
        \vspace{-0.2in}
            \begin{align*}
                \visible<8->{E[m_t] &= (1-\beta)\sum_{i=1}^{t}(\beta)^{t-i}E[g_i]}\\
                \visible<9->{ &= E[g](1-\beta)\color{red}\sum_{i=1}^t(\beta)^{t-i}}\\
                \visible<10->{&= E[g](1-\beta)\color{red}(\beta^{t-1} + \beta^{t-2} + \cdots + \beta^{0})} \\
                \visible<11->{&= E[g](1-\beta)\color{red}\frac{1 - \beta^t}{1 - \beta}}
            \end{align*}
            \visible<12->{the last fraction is the sum of a GP with common ratio = $\beta$}
            \begin{align*}
            	\visible<13->{E[m_t] &= E[g](1 - \beta^t)}\\
            	\visible<14->{E[\frac{m_t}{1 - \beta^t}] &= E[g]}\\
            	\visible<15->{E[\hat{m_t}] &= E[g] (\because \frac{m_t}{1 - \beta^t} = \hat{m_t})}
            \end{align*}
            	\visible<16->{Hence we apply the bias correction because then the expected value of $\hat{m_t}$ is the same as the expected value of $g_t$}
        \end{overlayarea}
    \end{columns}
\end{frame}


