\begin{frame}
	\myheading{Module 10.4: Continuous bag of words model}
\end{frame}

\begin{frame}
	\begin{block}{}
		\onslide<1-2>{
			\begin{itemize}\justifying
				\item<1-> The methods that we have seen so far are called \textbf{count based models} because they use the co-occurrence counts of words
				\item<2->We will now see methods which directly \textbf{learn} word representations (these are called \textbf{(direct) prediction based models})
			\end{itemize}}
	\end{block}
\end{frame}

\begin{frame}
	\begin{block}{The story ahead ...}
		\onslide<1->{
			\begin{itemize}
				\item<1-> Continuous bag of words model
				\item<2-> Skip gram model with negative sampling (the famous word2vec)
				\item<3-> GloVe word embeddings
				\item<4-> Evaluating word embeddings
				\item<5-> Good old SVD does just fine!!
			\end{itemize}}
	\end{block}
\end{frame}


\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\vspace{0.2in}
			\onslide<3->{
				\justifying
				\footnotesize{\textit{
						Sometime in the 21st century, Joseph Cooper, \colorbox{green}{\mbox{a widowed former engineer}} \colorbox{red!50}{and} former NASA pilot, runs a farm with his father-in-law Donald, son Tom, and daughter Murphy, It is post-truth society (\colorbox{green}{\mbox{Cooper is reprimanded for}} \colorbox{red!50}{telling} Murphy that the Apollo missions did indeed happen) and a series of crop blights threatens humanity's survival. \colorbox{green}{\mbox{Murphy believes her bedroom}} \colorbox{red!50}{is} haunted by a poltergeist. When a pattern is \colorbox{green}{\mbox{created out of dust}} \colorbox{red!50}{on} the floor, Cooper realizes that gravity is behind its formation, not a "ghost". He interprets the pattern as a \colorbox{green}{\mbox{set of geographic coordinates}} \colorbox{red!50}{formed} into binary code. Cooper and Murphy follow the coordinates to a secret NASA facility, where they are met by Cooper's former professor, Dr. Brand.
					}} \\~\\ \textbf{Some sample 4 word windows from a corpus}}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\onslide<1-5>{
				\begin{itemize}
					\justifying
					\item<1-> \textbf{Consider this Task:} Predict $n$-th word given previous $n$-1 words
					\item<2-> \textbf{Example:} he sat on a \textcolor{red}{chair}
					\item<3-> \textbf{Training data:} All $n$-word windows in your corpus
					\item<4-> Training data for this task is easily available (take all $n$ word windows from the whole of wikipedia)
					\item<5-> For ease of illustration, we will first focus on the case when $n=2$ (\textit{i.e.}, predict second word based on first word)
				\end{itemize}}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\begin{block}{}
		We will now try to answer these two questions:
		\onslide<1-3>{
			\begin{itemize}\justifying
				\item<2-> How do you model this task?
				\item<3-> What is the connection between this task and learning word representations?
			\end{itemize}}
	\end{block}
\end{frame}


\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\include{modules/Module4/tikz_images/wv}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\onslide<1-4>{
				\begin{itemize}
					\justifying
					\item<1-> We will model this problem using a feedforward neural network
					\item<2-> \textbf{Input:} One-hot representation of the \textbf{context word}
					\item<3-> \textbf{Output:} There are $|V|$ words (classes) possible and we want to predict a probability distribution over these $|V|$ classes (multi-class classification problem) 
					\item<4-> \textbf{Parameters:} $\mathbf{W}_{context} \in \mathbb{R}^{k \times |V|}$ and $\mathbf{W}_{word} \in \mathbb{R}^{k \times |V|}$\\
					      (we are assuming that the set of \textbf{words} and \textbf{context} words is the same: each of size $|V|$)
				\end{itemize}}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\include{modules/Module4/tikz_images/skip_gram}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
				\footnotesize{
					\begin{itemize}
						\justifying
						\item<1-> What is the product $\mathbf{W}_{context} \mathbf{x} $ given that $\mathbf{x}$ is a one hot vector
						\item<2-> It is simply the $i$-th column of $\mathbf{W}_{context}$
						      \begin{equation*}
							      \left[\begin{array}{c>{\columncolor{red!50}}cc}
								      -1 & 0.5 & 2  \\
								      3  & -1  & -2 \\
								      -2 & 1.7 & 3
							      \end{array}\right]
							      \begin{bmatrix}
								      0 \\
								      \colorbox{red!50}{1} \\
								      0
							      \end{bmatrix}
							      =\begin{bmatrix}
								      0.5 \\
								      -1  \\
								      1.7
							      \end{bmatrix}
						      \end{equation*}
						\item<3-> So when the $i^{th}$ word is present the $i^{th}$ element in the one hot vector is ON and the $i^{th}$ column of $\mathbf{W}_{context}$ gets selected
						\item<4-> In other words, there is a one-to-one correspondence between the words and the column of $\mathbf{W}_{context}$
						\item<5-> More specifically, we can treat the $i$-th column of $\mathbf{W}_{context}$ as the representation of context $i$
					\end{itemize}
				}
		\end{overlayarea}
	\end{columns}
\end{frame}


\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\include{modules/Module4/tikz_images/skip_gram}
			\vspace{-15mm}
			\onslide<3-> {
				\begin{align*}
					P(on|sat) = \frac{e^{(\mathbf{W}_{word}h)[i]}}{\sum_j e^{(\mathbf{W}_{word}h)[j]}}
				\end{align*}
			}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\footnotesize{
			\begin{itemize}
				\justifying
				\item<1-> How do we obtain $P(on|sat)$? For this multi-class classification problem what is an appropriate output function? \onslide<2-> {(\textbf{softmax})}
				\item <4-> Therefore, $P(on|sat)$ is proportional to the dot product between $j^{th}$ column of $\mathbf{W}_{context}$ and $i^{th}$ column of $\mathbf{W}_{word}$
				\item<5-> $P(word=i|sat)$ thus depends on the $i^{th}$ column of $\mathbf{W}_{word}$
				\item<6-> We thus treat the $i$-th column of $\mathbf{W}_{word}$ as the representation of word $i$
				\item<7-> Hope you see an analogy with SVD! (there we had a different way of learning $\mathbf{W}_{context}$ and $\mathbf{W}_{word}$ but we saw that the $i^{th}$ column of $\mathbf{W}_{word}$ corresponded to the representation of the $i^{th}$ word)
				\item<8-> Now that we understood the interpretation of $\mathbf{W}_{context}$ and $\mathbf{W}_{word}$, our aim now is to learn these parameters
					% \item<7-> Let us see how to learn these parameters
			\end{itemize}
			}
		\end{overlayarea}
	\end{columns}
\end{frame}


\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\include{modules/Module4/tikz_images/skip_gram1}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\footnotesize{
				\begin{itemize}
					\justifying
					\item<1-> We denote the context word (sat) by the index $c$ and the correct output word (on) by the index $w$
					\item<2-> For this multiclass classification problem what is an appropriate output function ($\hat{y} = f(x)$) ? \onslide<3->{\textbf{softmax}}
					\item<4-> What is an appropriate loss function? \onslide<5->{\textbf{cross entropy}}
					      \begin{align*}
						      \onslide<5->{\mathscr{L}(\theta) & = -\log \hat{y}_w = -\log P(w|c)\\}
						      \onslide<6->{h                   & = W_{context}\cdot x_c = u_c \\}
						      \onslide<7->{\hat{y}_w           & = \frac{exp(u_c \cdot v_{w})}{\sum_{w' \in V} exp(u_c \cdot v_{w'})}}
					      \end{align*}
					      \onslide<8->{$u_c$ is the column of $W_{context}$ corresponding to context $c$ and $v_{w}$ is the column of $W_{word}$ corresponding to context $w$}
				\end{itemize}}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\include{modules/Module4/tikz_images/skip_gram}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}
				\justifying
				\item<1-> How do we train this simple feed forward neural network? \onslide<2->{backpropagation}
				\item<3-> Let us consider one input-output pair $(c, w)$ and see the update rule for $v_w$
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\include{modules/Module4/tikz_images/skip_gram}
			\vspace{-2cm}
			\onslide<4>{
				\begin{align*}
					\nabla_{v_{w}} & = -\frac{\partial}{\partial v_w} \mathscr{L}(\theta)
				\end{align*}
			}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\justifying
			\begin{align*}
				\mathscr{L}(\theta)         & = -\log \hat{y}_w                                                                                                              \\
				\onslide<2->{               & = -\log \frac{exp(u_c \cdot {v}_{w})}{\sum_{w' \in V} exp(u_c \cdot {v}_{w'})} \\}
				\onslide<3->{               & = -(u_c \cdot {v}_{w} - \log \sum_{w' \in V} exp(u_c \cdot {v}_{w'})) \\}
				\onslide<4->{\nabla_{v_{w}} & = -({u}_{c} - \frac{exp(u_c \cdot {v}_{w})}{ \sum_{w' \in V} exp(u_c \cdot {v}_{w'})} \cdot {u}_{c}) \\}
				\onslide<5->{               & = - {u}_{c} (1 - \hat{y}_w)}
			\end{align*}
			\onslide<6->{And the update rule would be}
			\begin{align*}
				\onslide<7->{v_w & = v_w - \eta \nabla_{v_{w}} \\}
				\onslide<8->{    & = v_w + \eta {u}_{c} (1 - \hat{y}_w)}
			\end{align*}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\include{modules/Module4/tikz_images/skip_gram}
		\end{overlayarea}

		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\onslide<1->{
				\begin{itemize}
					\justifying
					\item This update rule has a nice interpretation
					      \begin{align*}
						      v_w & = v_w + \eta {u}_{c} (1 - \hat{y}_w)
					      \end{align*}
					\item<2-> If $\hat{y}_w \rightarrow 1$ then we are already predicting the right word and  $v_w$ will not be updated
					\item<3-> If $\hat{y}_w \rightarrow 0$ then $v_w$ gets updated by adding a fraction of $u_c$ to it
					\item<4-> This increases the cosine similarity between $v_w$ and $u_c$ (How? Refer to slide 38 of Lecture 2)
					\item<5-> The training objective ensures that the cosine similarity between word ($v_w$) and context word ($u_c$) is maximized
				\end{itemize}}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\include{modules/Module4/tikz_images/skip_gram}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\onslide<1-4>{
				\begin{itemize}
					\justifying
					\item<1-> What happens to the representations of two words $w$ and $w'$ which tend to appear in similar context ($c$)
					\item<2-> The training ensures that both $v_w$ and $v_w'$ have a high cosine similarity with $u_c$ and hence transitively (intuitively) ensures that $v_w$ and $v_w'$ have a high cosine similarity with each other
					\item<3-> This is only an intuition (reasonable)
					\item<4-> Haven't come across a formal proof for this!
				\end{itemize}}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\include{modules/Module4/tikz_images/word2vec_1}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\footnotesize{\begin{itemize}
				\justifying
				\item<1-> In practice, instead of window size of $1$ it is common to use a window size of $d$
				\item<2-> So now,
				 \vspace{-3mm}
				      \begin{align*}
					      h = \sum_{i=1}^{d-1} u_{c_{i}}
				      \end{align*}
				 \vspace{-3mm}
				 \item<3-> $[W_{context},W_{context}]$ just means that we are stacking 2 copies of $W_{context}$ matrix
				 \onslide<4-> {
				 \vspace{-3mm}
				 	\begin{align*}
				 		&\left[\begin{array}{c>{\columncolor{red!50}}cccc>{\columncolor{red!50}}c}
							-1 & 0.5 & 2 & \onslide<5->{-1 & 0.5 & 2}\\
							3 & -1 & -2 & \onslide<5->{3 & -1 & -2}\\
							-2 & 1.7 & 3 & \onslide<5->{-2 & 1.7 & 3}\\
						\end{array} \right]
						\begin{bmatrix}
							0\\
							\colorbox{red!50}{1}\\
							0\\
							0\\
							0\\
							\colorbox{red!50}{1}\\
						\end{bmatrix}
						&
						 \begin{array}[c]{@{}l@{}l}
						 \vphantom{0} \\
						\left. \vphantom{0}  \hspace{-0.5cm} \right\} \text{$sat$} \\
						 \vphantom{0}\\
						  \vphantom{0}\\
						   \vphantom{0}\\
						   \left. \vphantom{0} \hspace{-0.5cm}\right\} \text{$he$} \\
						   \end{array}\\
						\onslide<6->{&= \begin{bmatrix}
							2.5\\
							-3\\
							4.7
						\end{bmatrix}}
					 \end{align*}%
					 \vspace{-5mm}
				 }
				 \item<7-> The resultant product would simply be the sum of the columns corresponding to `sat' and `he'
			\end{itemize}}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\begin{itemize}
		\item<1-> Of course in practice we will not do this expensive matrix multiplication
		\item<2-> If `he' is $i^{th}$ word in the vocabulary and $sat$ is the $j^{th}$ word then we will simply access columns $\mathbf{W}[i:]$ and $\mathbf{W}[j:]$ and add them
	\end{itemize}
\end{frame}

\begin{frame}
	\begin{itemize}
		\item<1-> Now what happens during backpropagation
		\item<2-> Recall that
			\begin{align*}
				h = \sum_{i=1}^{d-1} u_{c_i}
			\end{align*}
		\item<3-> and 
			\begin{align*}
				P(on|sat,he) = \frac{e^{(w_{word}h)[k]}}{\sum_j e^{(w_{word}h)[j]}}
			\end{align*}
		\item<4-> where `k' is the index of the word `on'
		\item<5-> The loss function depends on $\{W_{word}, u_{c_1},u_{c_2},\ldots,u_{c_{d-1}}\}$ and all these parameters will get updated during backpropogation
		\item<6-> Try deriving the update rule for $v_w$ now and see how it differs from the one we derived before
	\end{itemize}
\end{frame}


\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\include{modules/Module4/tikz_images/word2vec_1}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\textbf{Some problems:}
			\onslide<1-3>{
				\begin{itemize}\justifying
					\item<1-> Notice that the softmax function at the output is computationally very expensive
					      \justifying
					      \begin{align*}
						      \hat{y}_w & = \frac{exp(u_c \cdot {v}_{w})}{\sum_{w' \in V} exp(u_c \cdot {v}_{w'})}
					      \end{align*}
					\item<2-> The denominator requires a summation over all words in the vocabulary
					\item<3-> We will revisit this issue soon
				\end{itemize}}
		\end{overlayarea}
	\end{columns}
\end{frame}


