\begin{frame}
	\myheading{Module 23.3: Generative Adversarial Networks - The Math Behind it}
\end{frame}

\begin{frame}
	\begin{itemize}\justifying
		\item<1-> We will now delve a bit deeper into the objective function used by GANs and see what it implies
		\item<2-> Suppose we denote the true data distribution by $p_{data}(x)$ and the distribution of the data generated by the model as $p_G(x)$
		\item<3-> What do we wish should happen at the end of training? 
					\visible<4->{ $$p_G(x) = p_{data}(x)$$ }
		\item<5-> Can we prove this formally even though the model is not explicitly computing this density?
		\item<6-> We will try to prove this over the next few slides
	\end{itemize}
\end{frame}

\begin{frame}
	\begin{block}{Theorem}
		The global minimum of the virtual training criterion $C(G)=\underset{D}{max} ~V(G,D)$ is achieved \textbf{if and only if} $p_G=p_{data}$\\
	\end{block}

	\begin{center}
		\visible<2->{\textbf{is equivalent to}}
	\end{center}

	\visible<3->
	{
		\begin{block}{Theorem}
			\begin{enumerate}
				\item \textbf{If} $p_G=p_{data}$ then the global minimum of the virtual training criterion $C(G)=\underset{D}{max} ~V(G,D)$ is achieved  \textbf{and}
				\item<4-> The global minimum of the virtual training criterion $C(G)=\underset{D}{max} ~V(G,D)$ is achieved \textbf{only if} $p_G=p_{data}$
			\end{enumerate}
		\end{block}
	}
	% \textbf{To Prove:}
	% \begin{itemize}[<+->]\justifying
	% 	\item The global minimum of the virtual training criterion $C(G)=\underset{D}{max} ~V(G,D)$ is achieved if and only if $p_G=p_{data}$
	% 	\item Any theorem containing an ``if and only if" condition can be split into two parts:
	% 	\begin{itemize}[<+->]\justifying
	% 		\item If $p_G=p_{data}$ then the global minimum of the virtual training criterion $C(G)=\underset{D}{max} ~V(G,D)$ is achieved
	% 		\item If the global minimum of the virtual training criterion $C(G)=\underset{D}{max} ~V(G,D)$ is achieved then $p_G=p_{data}$
	% 	\end{itemize}
	% \end{itemize}
\end{frame}

\begin{frame}
	\begin{block}{Outline of the Proof}
	\textbf{The `if' part:} The global minimum of the virtual training criterion $C(G)=\underset{D}{max} ~V(G,D)$ is achieved \textbf{if} $p_G=p_{data}$
		\begin{enumerate}
			\item[(a)]<2-> \alert<6>{Find the value of $V(D,G)$ when the generator is optimal \textit{i.e.}, when $p_G=p_{data}$}
			\item[(b)]<3-> Find the value of $V(D,G)$ for other values of the generator \textit{i.e.}, for any $p_G$ such that $p_G \neq p_{data}$
			\item[(c)]<4-> Show that $a < b$ $\forall$ $p_G \neq p_{data}$(and hence the minimum $V(D,G)$ is achieved when $p_G=p_{data}$)
		\end{enumerate}
	\vspace{5mm}
	\textbf{The `only if' part:} The global minimum of the virtual training criterion $C(G)=\underset{D}{max} ~V(G,D)$ is achieved \textbf{only if} $p_G=p_{data}$
	\begin{itemize}
		\item<5-> Show that when $V(D,G)$ is minimum then $p_G=p_{data}$
	\end{itemize}
	\end{block}
\end{frame}

\begin{frame}
	\begin{itemize}[<+->]
		\item First let us look at the objective function again 
				$$ \min\limits_{\phi}~ \max\limits_{\theta} \hspace*{4mm} [\mathbb{E}_{x \sim p_{data}} \log D_{\theta}(x) + \mathbb{E}_{z \sim p(z)} \log(1- D_{\theta}(G_{\phi}(z)) ) ] $$

		\item We will expand it to its integral form
			$$\min\limits_{\phi}~ \max\limits_{\theta} ~  \int_x p_{data}(x) \log D_{\theta}(x) + \int_z p(z) \log(1 - D_{\theta}(G_{\phi}(z)))$$
		\item Let $p_G(X)$ denote the distribution of the $X$'s generated by the generator and since $X$ is a function of $z$ we can replace the second integral as shown below
			$$\min\limits_{\phi}~ \max\limits_{\theta} ~  \int_x p_{data}(x) \log D_{\theta}(x) + \int_x p_G(x) \log(1 - D_{\theta}(x))$$
		% <replace the second interval involving z by x>
		\item The above replacement follows from the \textit{law of the unconscious statistician} \href{https://en.wikipedia.org/wiki/Law_of_the_unconscious_statistician}{(click to link of wikipedia page)}
	\end{itemize}
\end{frame}

\begin{frame}[shrink=10]
	\begin{itemize}[<+->]
		\item Okay, so our revised objective is given by
			$$\min\limits_{\phi}~ \max\limits_{\theta} ~ \int_x \left(p_{data}(x) \log D_{\theta}(x) +  p_G(x) \log(1 - D_{\theta}(x))\right)dx$$
		\item Given a generator G, we are interested in finding the optimum discriminator D which will maximize the above objective function
		\item The above objective will be maximized when the quantity inside the integral is maximized $\forall x$
		\item To find the optima we will take the derivative of the term inside the integral w.r.t. $D$ and set it to zero
			\begin{align*}
				\visible<5->{\frac{d}{d(D_\theta(x))}\left(p_{data}(x) \log D_{\theta}(x) +  p_{G}(x) \log(1 - D_{\theta}(x))\right) &= 0\\}
				\visible<6->{p_{data}(x) \frac{1}{D_{\theta}(x)} + p_{G}(x) \frac{1}{1-D_{\theta}(x)}(-1) &= 0\\}
				\visible<7->{\frac{p_{data}(x)}{D_{\theta}(x)} &=  \frac{p_{G}(x)}{1-D_{\theta}(x)} \\}
				\visible<8->{(p_{data}(x))(1-D_{\theta}(x)) &=  (p_{G}(x))(D_{\theta}(x)) \\}
				\visible<9->{D_\theta(x) &= \frac{p_{data}(x)}{p_{G}(x) + p_{data}(x)}}
			\end{align*}
		% <show the steps of the derivation ending with $D_\theta(x) = \frac{p_{data}}{p_{data} + p_{data}}$
	\end{itemize}
\end{frame}

\begin{frame}
	\begin{itemize}[<+->]
		\item This means for any given generator 
			$$D^{*}_G({G}(x)) = \frac{p_{data}(x)}{p_{data}(x) + p_G(x)}$$
		\item Now the if part of the theorem says ``if $p_G=p_{data}$ ...."
		\item So let us substitute $p_G = p_{data}$ into  $D^{*}_G({G}(x))$ and see what happens to the loss functions
		\begin{align*}
			\visible<4->{D^{*}_{{G}} &= \frac{p_{data}}{p_{data} + p_G} = \frac{1}{2}\\}
			\visible<5->{V({{G}},D^*_{{G}}) &= \int_x p_{data}(x) \log D(x) + p_G(x) \log\left(1-D(x)\right) dx\\  }
			\visible<6->{&= \int_x p_{data}(x) \log \frac{1}{2} + p_G(x) \log\left(1-\frac{1}{2}\right) dx\\}
			\visible<7->{&=  \log 2 \int_x p_G(x)dx - \log 2 \int_x p_{data}(x)dx		\\}
			\visible<8->{&= -2 \log 2 ~~~~~= -\log 4}
				    % &= <some trickery> see steps here https://srome.github.io/An-Annotated-Proof-of-Generative-Adversarial-Networks-with-Implementation-Notes\\
		\end{align*}
	\end{itemize}
\end{frame}

\begin{frame}
	\begin{block}{Outline of the Proof}
	\textbf{The `if' part:} The global minimum of the virtual training criterion $C(G)=\underset{D}{max} ~V(G,D)$ is achieved \textbf{if} $p_G=p_{data}$
		\begin{enumerate}
			\item[(a)] \color{gray}{Find the value of $V(D,G)$ when the generator is optimal \textit{i.e.}, when $p_G=p_{data}$} \color{black}
			\item[(b)] \alert<2>{Find the value of $V(D,G)$ for other values of the generator \textit{i.e.}, for any $p_G$ such that $p_G \neq p_{data}$}
			\item[(c)] Show that $a < b$ $\forall$ $p_G \neq p_{data}$(and hence the minimum $V(D,G)$ is achieved when $p_G=p_{data}$)
		\end{enumerate}
	\vspace{5mm}
	\textbf{The `only if' part:} The global minimum of the virtual training criterion $C(G)=\underset{D}{max} ~V(G,D)$ is achieved \textbf{only if} $p_G=p_{data}$
	\begin{itemize}
		\item Show that when $V(D,G)$ is minimum then $p_G=p_{data}$
	\end{itemize}
	\end{block}
\end{frame}

\begin{frame}
	\begin{itemize}[<+->]
		\item So what we have proved so far is that if the generator is optimal ($p_G = p_{data}$) the discriminator's loss value is $-\log 4$
		\item We still haven't proved that this is the minima
		\item For example, it is possible that for some $p_G \neq p_{data}$, the discriminator's loss value is lower than $-\log 4$
		\item To show that the discriminator achieves its lowest value ``if $p_G = p_{data}$", we need to show that for all other values of $p_G$ the discriminator's loss value is greater than $-\log 4$
	\end{itemize}
\end{frame}

\begin{frame}[shrink=25]
	\begin{itemize}[<+->]
		\item \scalebox{1.5}{To show this we will get rid of the assumption that $p_G=p_{data}$}
		% \item The entire derivation from the blog upto
		% $C(G)=-log4+KL(pdata||pdata+pG2)+KL(pG||pdata+pG2)$
	\end{itemize}
	\begin{align*}
		\visible<2->{C(G) &= \int_x \left[p_{data}(x) \log\left(\frac{p_{data}(x)}{p_G(x)+p_{data}(x)}\right) + p_G(x) \log\left(1-\frac{p_{data}(x)}{p_G(x)+p_{data}(x)}\right)\right]dx\\}
			\visible<3->{&= \int_x \left[p_{data}(x) \log\left(\frac{p_{data}(x)}{p_G(x)+p_{data}(x)}\right) + p_G(x) \log\left(\frac{p_G(x)}{p_G(x)+p_{data}(x)}\right) + (\log2-\log2)(p_{data}+p_{G})\right]dx\\}
			\visible<4->{&= -\log2 \int_x \left(p_G(x) + p_{data}(x)\right)dx \\
			&~~~~~+ \int_x \left[p_{data}(x) \left(\log 2+\log\left(\frac{p_{data}(x)}{p_G(x)+p_{data}(x)}\right)\right) 
					+ p_G(x)\left(\log2 + \log\left(\frac{p_G(x)}{P{p_G(x)+p_{data}(x)}}\right)\right)\right]dx \\}
			\visible<5->{&= -\log2 (1+1) \\
			&~~~~~+ \int_x \left[p_{data}(x) \log\left(\frac{p_{data}(x)}{\frac{p_G(x)+p_{data}(x)}{2}}\right) 
					+ p_G(x)\log\left(\frac{p_{G}(x)}{\frac{p_G(x)+p_{data}(x)}{2}}\right)\right]dx \\}
			\visible<6->{&= -\log4 + KL\left(p_{data}\|\frac{p_G(x)+p_{data}(x)}{2}\right) + KL\left(p_G\|\frac{p_G(x)+p_{data}(x)}{2}\right)}
	\end{align*}
\end{frame}

\begin{frame}
	\begin{block}{Outline of the Proof}
	\textbf{The `if' part:} The global minimum of the virtual training criterion $C(G)=\underset{D}{max} ~V(G,D)$ is achieved \textbf{if} $p_G=p_{data}$
		\begin{enumerate}
			\item[(a)] \color{gray}{Find the value of $V(D,G)$ when the generator is optimal \textit{i.e.}, when $p_G=p_{data}$} \color{black}
			\item[(b)] \color{gray}{Find the value of $V(D,G)$ for other values of the generator \textit{i.e.}, for any $p_G$ such that $p_G \neq p_{data}$} \color{black}
			\item[(c)] \alert<2>{Show that $a < b$ $\forall$ $p_G \neq p_{data}$(and hence the minimum $V(D,G)$ is achieved when $p_G=p_{data}$)}
		\end{enumerate}
	\vspace{5mm}
	\textbf{The `only if' part:} The global minimum of the virtual training criterion $C(G)=\underset{D}{max} ~V(G,D)$ is achieved \textbf{only if} $p_G=p_{data}$
	\begin{itemize}
		\item Show that when $V(D,G)$ is minimum then $p_G=p_{data}$
	\end{itemize}
	\end{block}
\end{frame}

\begin{frame}
	\begin{itemize}[<+->]
		\item Okay, so we have 
			$$C(G)=-\log 4 + KL\left(p_{data}||\frac{p_{data}+p_{g}}{2}\right)+KL\left(p_G||\frac{p_{data}+p_G}{2}\right)$$
		\item We know that KL divergence is always $\geq 0$
			$$\therefore C(G) \geq -\log 4$$
		\item Hence the minimum possible value of $C(G)$ is $-\log 4$ 
		\item But this is the value that $C(G)$ achieves when $p_G = p_{data}$ (and this is exactly what we wanted to prove)
		\item We have, thus, proved the \textbf{if part} of the theorem
	\end{itemize}
\end{frame}

\begin{frame}
	\begin{block}{Outline of the Proof}
	\textbf{The `if' part:} The global minimum of the virtual training criterion $C(G)=\underset{D}{max} ~V(G,D)$ is achieved \textbf{if} $p_G=p_{data}$
		\begin{enumerate}
			\item[(a)] \color{gray}{Find the value of $V(D,G)$ when the generator is optimal \textit{i.e.}, when $p_G=p_{data}$} \color{black}
			\item[(b)] \color{gray}{Find the value of $V(D,G)$ for other values of the generator \textit{i.e.}, for any $p_G$ such that $p_G \neq p_{data}$} \color{black}
			\item[(c)] \color{gray}{Show that $a < b$ $\forall$ $p_G \neq p_{data}$(and hence the minimum $V(D,G)$ is achieved when $p_G=p_{data}$)} \color{black}
		\end{enumerate}
	\vspace{5mm}
	\textbf{The `only if' part:} The global minimum of the virtual training criterion $C(G)=\underset{D}{max} ~V(G,D)$ is achieved \textbf{only if} $p_G=p_{data}$
	\begin{itemize}
		\item \alert<2>{Show that when $V(D,G)$ is minimum then $p_G=p_{data}$}
	\end{itemize}
	\end{block}
\end{frame}

\begin{frame}[shrink=15]
	\begin{itemize}[<+->]
		\item Now let's look at the other part of the theorem\\
			If the global minimum of the virtual training criterion $C(G)=\underset{D}{max}~V(G,D)$ is achieved then $p_G=p_{data}$
		\item We know that
				$$C(G)=-\log 4 + KL\left(p_{data}\|\frac{p_{data}+p_{g}}{2}\right)+KL\left(p_G\|\frac{p_{data}+p_G}{2}\right)$$
		\item If the global minima is achieved then $C(G) = -\log 4$ which implies that 
				$$ KL\left(p_{data}\|\frac{p_{data}+p_{g}}{2}\right)+KL\left(p_G\|\frac{p_{data}+p_G}{2}\right) = 0 $$
		\item This will happen only when $p_G = p_{data}$ (you can prove this easily)
		\item In fact $KL\left(p_{data}\|\frac{p_{data}+p_{g}}{2}\right)+KL\left(p_G\|\frac{p_{data}+p_G}{2}\right)$ is the Jenson-Shannon divergence between $p_G$ and $p_{data}$ 
				$$KL\left(p_{data}\|\frac{p_{data}+p_{g}}{2}\right)+KL\left(p_G\|\frac{p_{data}+p_G}{2}\right) = JSD(p_{data} \| p_G)$$
		\item[] which is minimum only when $p_G = p_{data}$
	\end{itemize}
\end{frame}