\begin{frame}
	\myheading{Module 2.5: Perceptron Learning Algorithm}
\end{frame}

\begin{frame}
	\begin{itemize}\justifying
		\item<1-> We will now see a more principled approach for learning these weights and threshold but before that let us answer this question...
		\item<2-> Apart from implementing boolean functions (which does not look very interesting) what can a perceptron be used for ?
		\item<3-> Our interest lies in the use of perceptron as a binary classifier. Let us see what this means...
	\end{itemize}
\end{frame}

\begin{frame}
	\begin{columns}
		\column{0.4\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{center}
				\onslide<5->{
					\input{modules/Module5/tikz_images/perceptron_params}
				}
			\end{center}
			\vspace{-0.3in}
			\only<4->{
				\begin{align*}
					x_1       & = isActorDamon                               \\
					x_2       & = isGenreThriller                            \\
					x_3       & = isDirectorNolan                            \\
					x_4       & = imdbRating \textit{(scaled to 0 to 1)}     \\
					... \quad & \quad...                                     \\
					x_n       & = criticsRating  \textit{(scaled to 0 to 1)}
				\end{align*}
			}
		\end{overlayarea}
		\column{0.6\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}\justifying
				\item<1-> Let us reconsider our problem of deciding whether to watch a movie or not
				\item<2-> Suppose we are given a list of $m$ movies and a label (class) associated with each movie indicating whether the user liked this movie or not : binary decision
				\item<3-> Further, suppose we represent each movie with $n$ features (some boolean, some real valued)
				\item<5-> We will assume that the data is linearly separable and we want a perceptron to learn how to make this decision
				\item<6-> In other words, we want the perceptron to find the equation of this separating plane (or find the values of $w_0, w_1, w_2, .., w_m$)
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}


\begin{frame}
	\begin{columns}

		\column{0.55\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{center}
				\begin{algorithm}[H]
					\onslide<2->{$P \leftarrow inputs\quad with \quad label \quad 1$\;}
					\onslide<3->{$N \leftarrow inputs \quad with \quad label \quad 0$\;}
					\onslide<4->{Initialize $\mathbf{w}$ randomly\;}
					\onslide<5->{\While{$!convergence$}{
							\onslide<7->{Pick random $\mathbf{x} \in P \cup N$ \;}
							\onslide<8->{\If{$\mathbf{x} \in P \quad and \quad \sum_{i=0}^{n} w_i * x_i < 0$} {
									\onslide<9->{$\mathbf{w} = \mathbf{w} + \mathbf{x}$ \;}
								}}
							\onslide<10->{\If{$\mathbf{x} \in N \quad and \quad \sum_{i=0}^{n} w_i * x_i \geq 0$} {
									\onslide<11->{$\mathbf{w} = \mathbf{w} - \mathbf{x}$ \;}
								}}
						}}
					\onslide<6->{//the algorithm converges when all the inputs are classified correctly}
					\caption{Perceptron Learning Algorithm}
				\end{algorithm}
			\end{center}
		\end{overlayarea}
		\column{0.45\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}\justifying
				\item<12-> Why would this work ?
				\item<13-> To understand why this works we will have to get into a bit of Linear Algebra and a bit of geometry...
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}


\begin{frame}
	\begin{columns}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}\justifying
				\item<1-> Consider two vectors $\mathbf{w}$ and $\mathbf{x}$
				      \only<2->{
					      \begin{align*}
						      \mathbf{w}                     & = [w_0, w_1, w_2, ..., w_n]               \\
						      \mathbf{x}                     & = [1, x_1, x_2, ..., x_n]                 \\
						      \onslide<3->{\mathbf{w\cdot x} & = \mathbf{w^Tx} = \sum_{i=0}^{n} w_i*x_i}
					      \end{align*}
				      }
				\item<4-> We can thus rewrite the perceptron rule as
				      \only<5->{
					      \begin{align*}
						      y & = 1 \quad if \quad \mathbf{w^Tx} \geq 0 \\
						        & = 0 \quad if \quad \mathbf{w^Tx} < 0
					      \end{align*}
				      }
			\end{itemize}
		\end{overlayarea}
		\column{0.5\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}\justifying
				\item<6-> We are interested in finding the line $\mathbf{w^Tx} = 0$ which divides the input space into two halves
				\item<7-> Every point ($\mathbf{x}$) on this line satisfies the equation $\mathbf{w^Tx} = 0$
				\item<8-> What can you tell about the angle ($\alpha$) between $\mathbf{w}$ and any point ($\mathbf{x}$) which lies on this line ?
				\item<9-> The angle is 90$\degree$ ($\because cos \alpha = \frac{w^Tx}{||w||||x||} = 0$)
				\item<10-> Since the vector $\mathbf{w}$ is perpendicular to every point on the line it is actually perpendicular to the line itself
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\begin{columns}
		\column{0.6\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}\justifying
				\item<2-> Consider some points (vectors) which lie in the positive half space of this line (\textit{i.e.}, $\mathbf{w^Tx} \geq 0$)
				\item<3-> What will be the angle between any such vector and $\mathbf{w}$ ? \only<4->{Obviously, less than 90$\degree$}
				\item<5-> What about points (vectors) which lie in the negative half space of this line (\textit{i.e.}, $\mathbf{w^Tx} < 0$)
				\item<6-> What will be the angle between any such vector and $\mathbf{w}$ ? \only<7->{Obviously, greater than 90$\degree$}
				\item<8-> Of course, this also follows from the formula ($cos \alpha = \frac{w^Tx}{||w||||x||}$)
				\item<9-> Keeping this picture in mind let us revisit the algorithm
			\end{itemize}
		\end{overlayarea}
		\column{0.4\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{center}
				\input{modules/Module5/tikz_images/plot_pos_neg}
			\end{center}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\begin{columns}
		\column{0.55\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{center}
				\begin{algorithm}[H]
					$P \leftarrow inputs\quad with \quad label \quad 1$\;
					$N \leftarrow inputs \quad with \quad label \quad 0$\;
					Initialize $\mathbf{w}$ randomly\;
					\While{$!convergence$}{
						Pick random $\mathbf{x} \in P \cup N$ \;
						\If{$\mathbf{x} \in P \quad and \quad \mathbf{w.x} < 0$} {
							$\mathbf{w} = \mathbf{w} + \mathbf{x}$ \;
						}
						\If{$\mathbf{x} \in N \quad and \quad \mathbf{w.x} \geq 0$} {
							$\mathbf{w} = \mathbf{w} - \mathbf{x}$ \;
						}
					}
					//the algorithm converges when all the inputs are classified correctly
					\caption{Perceptron Learning Algorithm}
				\end{algorithm}
				\vspace{-0.2in}
				\scriptsize{
					\begin{align*}
						cos \alpha = \frac{\mathbf{w}^T\mathbf{x}}{||\mathbf{w}||||\mathbf{x}||}
					\end{align*}
				}
			\end{center}
		\end{overlayarea}
		\column{0.45\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}\justifying
				\item<2-> For $\mathbf{x} \in P$ if $\mathbf{w.x} < 0$ then it means that the angle ($\alpha$) between this $\mathbf{x}$ and the current $\mathbf{w}$ is greater than 90$\degree$ \onslide<3->{(but we want $\alpha$ to be less than 90$\degree$)}
				\item<4-> What happens to the new angle ($\alpha_{new}$) when $\mathbf{w_{new}} = \mathbf{w} + \mathbf{x}$
				      \onslide<5-> {
					      \begin{align*}
						      \onslide<5-> {cos (\alpha_{new}) & \propto \mathbf{w_{new}}^{T}\mathbf{x} \\}
						      \onslide<6-> {                   & \propto (\mathbf{w} + \mathbf{x})^T\mathbf{x} \\}
						      \onslide<7-> {                   & \propto \mathbf{w}^T\mathbf{x} + \mathbf{x}^T\mathbf{x} \\}
						      \onslide<8-> {                   & \propto cos \alpha + \mathbf{x}^T\mathbf{x} \\}
						      \onslide<9-> {cos (\alpha_{new}) & > cos \alpha  }
					      \end{align*}
				      }
				      \vspace{-0.35in}
				\item<10-> Thus $\alpha_{new}$ will be less than $\alpha$ and this is exactly what we want
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\begin{columns}
		\column{0.55\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{center}
				\begin{algorithm}[H]
					$P \leftarrow inputs\quad with \quad label \quad 1$\;
					$N \leftarrow inputs \quad with \quad label \quad 0$\;
					Initialize $\mathbf{w}$ randomly\;
					\While{$!convergence$}{
						Pick random $\mathbf{x} \in P \cup N$ \;
						\If{$\mathbf{x} \in P \quad and \quad \mathbf{w.x} < 0$} {
							$\mathbf{w} = \mathbf{w} + \mathbf{x}$ \;
						}
						\If{$\mathbf{x} \in N \quad and \quad \mathbf{w.x} \geq 0$} {
							$\mathbf{w} = \mathbf{w} - \mathbf{x}$ \;
						}
					}
					//the algorithm converges when all the inputs are classified correctly
					\caption{Perceptron Learning Algorithm}
				\end{algorithm}
				\vspace{-0.2in}
				\scriptsize{
					\begin{align*}
						cos \alpha = \frac{\mathbf{w}^T\mathbf{x}}{||\mathbf{w}||||\mathbf{x}||}
					\end{align*}
				}
			\end{center}
		\end{overlayarea}
		\column{0.45\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}\justifying
				\item<2-> For $\mathbf{x} \in N$ if $\mathbf{w.x} \geq 0$ then it means that the angle ($\alpha$) between this $\mathbf{x}$ and the current $\mathbf{w}$ is less than 90$\degree$ \onslide<3->{(but we want $\alpha$ to be greater than 90$\degree$)}
				\item<4-> What happens to the new angle ($\alpha_{new}$) when $\mathbf{w_{new}} = \mathbf{w} - \mathbf{x}$
				      \onslide<5-> {
					      \begin{align*}
						      \onslide<5-> {cos (\alpha_{new}) & \propto \mathbf{w_{new}}^{T}\mathbf{x} \\}
						      \onslide<6-> {                   & \propto (\mathbf{w} - \mathbf{x})^T\mathbf{x} \\}
						      \onslide<7-> {                   & \propto \mathbf{w}^T\mathbf{x} - \mathbf{x}^T\mathbf{x} \\}
						      \onslide<8-> {                   & \propto cos \alpha - \mathbf{x}^T\mathbf{x} \\}
						      \onslide<9-> {cos (\alpha_{new}) & < cos \alpha  }
					      \end{align*}
				      }
				      \vspace{-0.35in}
				\item<10-> Thus $\alpha_{new}$ will be greater than $\alpha$ and this is exactly what we want
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}

\begin{frame}
	\begin{itemize}\justifying
		\item<1-> We will now see this algorithm in action for a toy dataset
	\end{itemize}
\end{frame}

\begin{frame}
	\begin{columns}
		\column{0.4\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{center}
				\input{modules/Module5/tikz_images/positive_negative}
			\end{center}
		\end{overlayarea}
		\column{0.6\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
			\begin{itemize}\justifying
				\item<1-> We initialized $\mathbf{w}$ to a random value
				\item<2-> We observe that currently, $\mathbf{w}\cdot \mathbf{x} < 0$ ($\because$ angle $>$ 90$\degree$) for all the positive points and  $ \mathbf{w} \cdot \mathbf{x} \geq 0$ ($\because$ angle $<$ 90$\degree$) for all the negative points (the situation is exactly oppsite of what we actually want it to be)
				\item<3-> We now run the algorithm by randomly going over the points
				      \only<4-5>{\item Randomly pick a point (say, $p_1$), apply correction $\mathbf{w} = \mathbf{w} + \mathbf{x}$ $\because \mathbf{w\cdot x < 0}$ (you can check the angle visually) }
				      \only<6-7>{\item Randomly pick a point (say, $p_2$), apply correction $\mathbf{w} = \mathbf{w} + \mathbf{x}$ $\because \mathbf{w\cdot x < 0}$ (you can check the angle visually) }
				      \only<8-9>{\item Randomly pick a point (say, $n_1$), apply correction $\mathbf{w} = \mathbf{w} - \mathbf{x}$ $\because \mathbf{w\cdot x \geq 0}$ (you can check the angle visually) }
				      \only<10-11>{\item Randomly pick a point (say, $n_3$), no correction needed $\because \mathbf{w\cdot x < 0}$ (you can check the angle visually) }
				      \only<12-13>{\item Randomly pick a point (say, $n_2$), no correction needed $\because \mathbf{w\cdot x < 0}$ (you can check the angle visually)}
				      \only<14-15>{\item Randomly pick a point (say, $p_3$), apply correction $\mathbf{w} = \mathbf{w} + \mathbf{x}$ $\because \mathbf{w\cdot x < 0}$ (you can check the angle visually) }
				      \only<16-17>{\item Randomly pick a point (say, $p_1$), no correction needed $\because \mathbf{w\cdot x \geq 0}$ (you can check the angle visually)}
				      \only<18-19>{\item Randomly pick a point (say, $p_2$), no correction needed $\because \mathbf{w\cdot x \geq 0}$ (you can check the angle visually)}
				      \only<20-21>{\item Randomly pick a point (say, $n_1$), no correction needed $\because \mathbf{w\cdot x < 0}$ (you can check the angle visually)}
				      \only<22-23>{\item Randomly pick a point (say, $n_3$), no correction needed $\because \mathbf{w\cdot x < 0}$ (you can check the angle visually)}
				      \only<24-25>{\item Randomly pick a point (say, $n_2$), no correction needed $\because \mathbf{w\cdot x < 0}$ (you can check the angle visually)}
				      \only<26>{\item Randomly pick a point (say, $p_3$), no correction needed $\because \mathbf{w\cdot x \geq 0}$ (you can check the angle visually)}
				\item<27> The algorithm has converged
			\end{itemize}
		\end{overlayarea}
	\end{columns}
\end{frame}
