\begin{frame}
	\myheading{Module 10.8: GloVe representations}
\end{frame}

\begin{frame}
	\begin{itemize}\justifying
		\item<1-> \textbf{Count} based methods (SVD) rely on global co-occurrence counts from the corpus for computing word representations
		\item<2-> Predict based methods \textbf{learn} word representations using co-occurrence information
		\item<3-> Why not combine the two (\textbf{count} and \textbf{learn}) ?
	\end{itemize}
\end{frame}


\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\large{\textbf{Corpus:}} \\
			\tiny{\begin{itemize}\justifying
					\item Human machine interface for computer applications\\
					\item User opinion of computer system response time\\
					\item User interface management system\\
					\item System engineering for improved response time\\
				\end{itemize}}
			\vspace{-0.2in}
			\tiny{\begin{table}
					\caption*{$X=$}
					\begin{tabular}{|>{\centering\arraybackslash}p{0.9cm}|>{\centering\arraybackslash}p{0.7cm}|>{\centering\arraybackslash}p{0.87cm}|>{\centering\arraybackslash}p{0.68cm}|>{\centering\arraybackslash}p{0.55cm}|>{\centering\arraybackslash}p{0.2cm}|>{\centering\arraybackslash}p{0.55cm}|}
						\hline
							  & \textbf{human} & \textbf{machine} & \textbf{system\hspace{0.2cm}} & \textbf{for} & ... & \textbf{user} \\
						\hline

						\textbf{human}   & 2.01           & 2.01             & 0.23            & 2.14         & ... & 0.43          \\
						\textbf{machine} & 2.01           & 2.01             & 0.23            & 2.14         & ... & 0.43          \\
						\textbf{system}  & 0.23           & 0.23             & 1.17            & 0.96         & ... & 1.29          \\
						\textbf{for}     & 2.14           & 2.14             & 0.96            & 1.87         & ... & -0.13         \\
						.                & .              & .                & .               & .            & .   & .             \\
						.                & .              & .                & .               & .            & .   & .             \\
						.                & .              & .                & .               & .            & .   & .             \\
						\textbf{user}    & 0.43           & 0.43             & 1.29            & -0.13        & ... & 1.71          \\
						\hline
					\end{tabular}
				\end{table}}

			\footnotesize{
				\begin{align*}
					P(j|i) & = \frac{X_{ij}}{\sum X_{ij}} =\frac{X_{ij}}{X_{i}} \\
					X_{ij} & = X_{ji}
				\end{align*}
			}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\footnotesize{
				\begin{itemize}
					\justifying
					\item<1-> $X_{ij}$ encodes important global information
					      about the co-occurrence between $i$ and $j$ (global: because it is computed for the entire corpus)
					\item<2-> Why not learn word vectors which are faithful
					      to this information?
					\item<3-> For example, enforce
					      \begin{align*}
						      v_{i}^{T}v_j & = \log P(j|i)             \\
						                   & = \log X_{ij} - \log(X_i)
					      \end{align*}
					\item<4-> Similarly,
					      \begin{align*}
						      v_{j}^{T}v_i & = \log X_{ij} - \log X_j \quad (X_{ij} = X_{ji})
					      \end{align*}
					\item<5-> Essentially we are saying that we want word vectors $v_i$ and $v_j$ such that $v_i^T v_j$ is faithful to the globally computed $P(j|i)$
				\end{itemize}
			}
		\end{overlayarea}
	\end{columns}
\end{frame}


\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\large{\textbf{Corpus:}} \\
			\tiny{\begin{itemize}\justifying
					\item Human machine interface for computer applications\\
					\item User opinion of computer system response time\\
					\item User interface management system\\
					\item System engineering for improved response time\\
				\end{itemize}}
			\vspace{-0.2in}
			\tiny{\begin{table}
					\caption*{$X=$}
					\begin{tabular}{|>{\centering\arraybackslash}p{0.9cm}|>{\centering\arraybackslash}p{0.7cm}|>{\centering\arraybackslash}p{0.87cm}|>{\centering\arraybackslash}p{0.68cm}|>{\centering\arraybackslash}p{0.55cm}|>{\centering\arraybackslash}p{0.2cm}|>{\centering\arraybackslash}p{0.55cm}|}
						\hline
							  & \textbf{human} & \textbf{machine} & \textbf{system\hspace{0.2cm}} & \textbf{for} & ... & \textbf{user} \\
						\hline

						\textbf{human}   & 2.01           & 2.01             & 0.23            & 2.14         & ... & 0.43          \\
						\textbf{machine} & 2.01           & 2.01             & 0.23            & 2.14         & ... & 0.43          \\
						\textbf{system}  & 0.23           & 0.23             & 1.17            & 0.96         & ... & 1.29          \\
						\textbf{for}     & 2.14           & 2.14             & 0.96            & 1.87         & ... & -0.13         \\
						.                & .              & .                & .               & .            & .   & .             \\
						.                & .              & .                & .               & .            & .   & .             \\
						.                & .              & .                & .               & .            & .   & .             \\
						\textbf{user}    & 0.43           & 0.43             & 1.29            & -0.13        & ... & 1.71          \\
						\hline
					\end{tabular}
				\end{table}}

			\footnotesize{
				\begin{align*}
					P(j|i) & = \frac{X_{ij}}{\sum X_{ij}} =\frac{X_{ij}}{ X_{i}} \\
					X_{ij} & = X_{ji}
				\end{align*}
			}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\onslide<1-3>{
				\footnotesize{
					\begin{itemize}
						\justifying
						\item<1-> Adding the two equations we get
						      \begin{align*}
							      2v_{i}^{T}v_j & = 2\log X_{ij} - \log X_{i} - \log X_{j}                  \\
							      v_{i}^{T}v_j  & = \log X_{ij} - \frac{1}{2}\log X_i - \frac{1}{2}\log X_j
						      \end{align*}
						\item<2-> Note that $\log X_i$ and $\log X_j$ depend only on the words $i$ \& $j$
						      and we can think of them as word specific biases which will be learned
						      \begin{align*}
							      v_i^{T}v_j             & = \log X_{ij} - b_i - b_j \\
							      v_i^{T}v_j + b_i + b_j & = \log X_{ij}
						      \end{align*}
						\item<3-> We can then formulate this as the following optimization problem
						      \begin{align*}
							      \underset{v_i,v_j,b_i,b_j}{min}\sum\limits_{i,j}(\underbrace{v_{i}^{T}v_j + b_i + b_j}_{\substack{\text{predicted value}\\ \text{using model}\\ \text{parameters}}} -\underbrace{\log X_{ij}}_{\substack{\text{actual value}\\ \text{computed from}\\ \text{the given corpus}}})^{2}
						      \end{align*}
					\end{itemize}}}
		\end{overlayarea}
	\end{columns}
\end{frame}



\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\large{\textbf{Corpus:}} \\
			\tiny{\begin{itemize}\justifying
					\item Human machine interface for computer applications\\
					\item User opinion of computer system response time\\
					\item User interface management system\\
					\item System engineering for improved response time\\
				\end{itemize}}
			\vspace{-0.2in}
			\tiny{\begin{table}
					\caption*{$X=$}
					\begin{tabular}{|>{\centering\arraybackslash}p{0.9cm}|>{\centering\arraybackslash}p{0.7cm}|>{\centering\arraybackslash}p{0.87cm}|>{\centering\arraybackslash}p{0.68cm}|>{\centering\arraybackslash}p{0.55cm}|>{\centering\arraybackslash}p{0.2cm}|>{\centering\arraybackslash}p{0.55cm}|}
						\hline
							  & \textbf{human} & \textbf{machine} & \textbf{system\hspace{0.2cm}} & \textbf{for} & ... & \textbf{user} \\
						\hline

						\textbf{human}   & 2.01           & 2.01             & 0.23            & 2.14         & ... & 0.43          \\
						\textbf{machine} & 2.01           & 2.01             & 0.23            & 2.14         & ... & 0.43          \\
						\textbf{system}  & 0.23           & 0.23             & 1.17            & 0.96         & ... & 1.29          \\
						\textbf{for}     & 2.14           & 2.14             & 0.96            & 1.87         & ... & -0.13         \\
						.                & .              & .                & .               & .            & .   & .             \\
						.                & .              & .                & .               & .            & .   & .             \\
						.                & .              & .                & .               & .            & .   & .             \\
						\textbf{user}    & 0.43           & 0.43             & 1.29            & -0.13        & ... & 1.71          \\
						\hline
					\end{tabular}
				\end{table}}

			\footnotesize{
				\begin{align*}
					P(j|i) & = \frac{X_{ij}}{\sum X_{ij}} =\frac{X_{ij}}{\sum X_{i}} \\
					X_{ij} & = X_{ji}
				\end{align*}
			}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\footnotesize{\vspace{0.1in}
				\begin{align*}
					\underset{v_i,v_j,b_i,b_j}{min}\sum\limits_{i,j}(v_{i}^{T}v_j + b_i + b_j -\log X_{ij})^{2}
				\end{align*}
				\onslide<2-4>{
					\begin{itemize}
						\justifying
						\item<2-> \textbf{Drawback:} weighs all co-occurrences equally
						\item<3-> \textbf{Solution:} add a weighting function
						      \begin{align*}
							      \underset{v_i,v_j,b_i,b_j}{min}\sum\limits_{i,j}f(X_{ij})(v_{i}^{T}v_j + b_i + b_j -\log X_{ij})^{2}
						      \end{align*}
						\item<4-> \textbf{Wishlist:} $f(X_{ij})$ should be such that neither rare nor frequent
						      words are overweighted.
						      \begin{align*}
							      f(x) = \left\{\begin{array}{lr}
								      (\frac{x}{x_{max}})^{\alpha}, & \text{if } x < x_{max} \\
								      1,                            & \text{otherwise}       \\
							      \end{array}\right\}
						      \end{align*}
						      where $\alpha$ can be tuned for a given dataset
					\end{itemize}}}
		\end{overlayarea}
	\end{columns}
\end{frame}
