\begin{frame}
	\myheading{Module 20.1 : Markov Chains}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%5
\begin{frame}
			
	\begin{columns}
				
		\begin{column}{0.4\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\centering
				\vspace{1cm}
				\onslide<2->{
					\begin{figure}
						\includegraphics[scale=0.4]{images/noisy_image.jpg}
						\caption{$X\in R^{1024}$}
					\end{figure}
				}
				\vspace{1cm}
				\onslide<3->{				
					\begin{align*}
						\mathbb{E}_{P(X)}[f(X)]
					\end{align*}	
				}
			\end{overlayarea}
		\end{column}
		\begin{column}{0.6\textwidth}
			
			
			\begin{itemize}
				\justifying
				%\setlength\itemsep{1em}
				\item<1-> Let us first begin by restating our goals
				\item<2-> \textbf{Goal 1}: Given a random variable $X\in R^n$, we are interested in drawing samples from the joint distribution $P(\textbf{X}$) 
				\item<3-> \textbf{Goal 2}: Given a function $f(X)$ defined over the random variable X, we are interested in computing the expectation $\mathbb{E}_{P(X)}[f(X)]$
				\item<4-> We will use Gibbs Sampling (class of Metropolis-Hastings algorithm) to achieve these goals 
				\item<5-> We will first understand the intuition behind Gibbs Sampling and then understand the math behind it
			\end{itemize}
		\end{column}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%55

\begin{frame}
	\begin{columns}
				
		\begin{column}{0.4\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\centering
				\vspace{1cm}
				\begin{figure}
					\includegraphics[scale=0.4]{images/noisy_image.jpg}
					\caption{$X\in R^{1024}$}
				\end{figure}
				\vspace{1cm}				
				\begin{align*}
					\mathbb{E}_{P(X)}[f(X)]
				\end{align*}	
				
			\end{overlayarea}
		\end{column}
		\begin{column}{0.6\textwidth}
			
			
			\begin{itemize}
				\justifying
				\item<1-> Suppose instead of a single random variable $X\in R^n$, we have a chain of random variables $X_1, X_2, \dots, X_K$ each $X_i\in R^n$ 
				\item<2-> The $i$ here corresponds to a time step
				\item<3-> For example, $X_i$ could be a n-dimensional vector containing the number of customers in a given set of $n$ restaurants on day $i$
				\item<4-> In our case, $X_i$ could be a 1024 dimensional image sent by our friend on day $i$
				\item<5-> For ease of illustration we will stick to the restaurant example and assume that instead of actual counts we are interested only in binary counts (high=1, low=0)
				\item<6-> Thus $X_i \in \{0, 1\}^n$
			\end{itemize}
		\end{column}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%55

\begin{frame}
	\begin{columns}
				
		\begin{column}{0.4\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
	            \only<1->{
				\vspace{3.5cm}
				\tikzstyle{neuronv}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=orange!30, draw=red!50]
				\tikzstyle{neuronh}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=blue!20, draw=blue!60]
				\tikzstyle{stateTransition}=[thick]
				\tikzstyle{learned}=[text=black]
				\begin{center}
					\begin{tikzpicture}[scale=1.9]
					    \onslide<1->{
					    	\node (h1)[neuronh] at (0, 1.5) {$x_1$};
					    }
					    \onslide<2->{
					    	\node (h2)[neuronh] at (0.7, 1.5) {$x_2$};
					    }
					    \onslide<4->{
					    	\node (h3)[neuronh] at (1.4, 1.5) {$x_3$};
						}
		
					    \onslide<3->{
					    	\draw [->] (h1)	-- (h2);
					    }
					    \onslide<4->{
					    	\draw [->] (h2)	-- (h3);
					    }
		
					\end{tikzpicture}
				\end{center}
			}
			\end{overlayarea}

		\end{column}
		\begin{column}{0.6\textwidth}
			
			
			\begin{itemize}
				\justifying
				\item<1-> On day 1, let $X_1$ take on the value $x_1$ ($x_1$ is one of the possible $2^n$ vectors) 
				\item<2-> On day 2, let $X_2$ take on the value $x_2$ ($x_2$ is again one of the possible $2^n$ vectors) 
				\item<3-> One way of looking at this is that the state has transitioned from $x_1$ to $x_2$
				\item<4-> Similarly, on day 3, if $X_3$ takes on the value $x_3$ then we can say that the state has transitioned from $x_1$ to $x_2$ to $x_3$ 
				\item<5-> Finally, on day $n$, we can say that the state has transitioned from $x_1$ to $x_2$ to $x_3$ to $ \dots x_n$
			\end{itemize}
		\end{column}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%55

\begin{frame}
	\begin{columns}
				
		\begin{column}{0.3\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\vspace{3.5cm}
				\tikzstyle{neuronv}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=orange!30, draw=red!50]
				\tikzstyle{neuronh}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=blue!20, draw=blue!60]
				\tikzstyle{stateTransition}=[thick]
				\tikzstyle{learned}=[text=black]
				\begin{center}
					\begin{tikzpicture}[scale=1.5]
					    \node (h1)[neuronh] at (0, 1.5) {$x_1$};
					    \node (h2)[neuronh] at (0.7, 1.5) {$x_2$};
					    \node (h3)[neuronh] at (1.4, 1.5) {$x_3$};
					    \node (h4)[] at (2.1, 1.5) {$\cdots$};
					    \node (h5)[neuronh] at (2.8, 1.5) {$x_i$};
					
					    \draw [->] (h1)	-- (h2);
					    \draw [->] (h2)	-- (h3);
					    \draw [->] (h3)	-- (h4);
					    \draw [->] (h4)	-- (h5);
		
					\end{tikzpicture}
				\end{center}
			
			\end{overlayarea}
		
		\end{column}
		\begin{column}{0.7\textwidth}
			
			
			\begin{itemize}
				\justifying
				\item<1-> We may now be interested in knowing what is the most likely value that the state will take on day $i$ given the states on day 1 to day $i-1$ 
				\item<2-> More formally, we may be interested in the following distribution
					\vspace{-1em}
					\begin{align*}
						P(X_i = x_i | X_1=x_1, X_2=x_2, \dots, X_{i-1}=x_{i-1})
					\end{align*}
				
				\item<3-> Now suppose the chain exhibits the following Markov property
					\vspace{-1em}
					\begin{align*}
						P(X_i = x_i | X_1=x_1, X_2=x_2, \dots, X_{i-1}=x_{i-1})\\ 
						= P(X_i = x_i | X_{i-1}=x_{i-1})
					\end{align*}
				\item<4-> In other words, given the previous state $X_{i-1}$, $X_i$ is independent of all preceding states 
				\item<5-> Can we draw a graphical model to encode this independence assumption ?
			\end{itemize}
		\end{column}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%55

\begin{frame}
	\begin{columns}
				
		\begin{column}{0.4\textwidth}
		\begin{overlayarea}{\textwidth}{\textheight}
            \only<2->{
			\vspace{3.5cm}
			\tikzstyle{neuronv}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=orange!30, draw=red!50]
			\tikzstyle{neuronh}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=blue!20, draw=blue!60]
			\tikzstyle{stateTransition}=[thick]
			\tikzstyle{learned}=[text=black]
			\begin{center}
				\begin{tikzpicture}[scale=1.9]
				    \node (h1)[neuronh] at (0, 1.5) {$X_1$};
				    \node (h2)[neuronh] at (0.7, 1.5) {$X_2$};
				    \node (h3)[] at (1.4, 1.5) {$\cdots$};
				    \node (h4)[neuronh] at (2.1, 1.5) {$X_k$};
	
	
				    \onslide<4->{
				    \draw [->] (h1)	-- (h2);
				    \draw [->] (h2)	-- (h3);
				    \draw [->] (h3)	-- (h4);
				    }
	
				\end{tikzpicture}
			\end{center}
		}
		\end{overlayarea}

		\end{column}
		\begin{column}{0.6\textwidth}
			
			
			\begin{itemize}
				\justifying
				\item<1-> In this graphical model, the random variables are $X_1, X_2, \dots, X_k$
				\item<2-> We will have a node corresponding to each of these random variables
				\item<3-> What will be the edges in the graph ?
				\item<4-> Well, each node only depends on its predecessor, so we will just have an edge between successive nodes
			\end{itemize}
		\end{column}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%55

\begin{frame}
	\begin{columns}		
		\begin{column}{0.4\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\vspace{3.5cm}
				\tikzstyle{neuronv}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=orange!30, draw=red!50]
				\tikzstyle{neuronh}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=blue!20, draw=blue!60]
				\tikzstyle{stateTransition}=[thick]
				\tikzstyle{learned}=[text=black]
				\begin{center}
					\begin{tikzpicture}[scale=1.9]
						\node (h1)[neuronh] at (0, 1.5) {$X_1$};
						\node (h2)[neuronh] at (0.7, 1.5) {$X_2$};
						\node (h3)[] at (1.4, 1.5) {$\cdots$};
						\node (h4)[neuronh] at (2.1, 1.5) {$X_k$};
									
									
						\draw [->] (h1)	-- (h2);
						\draw [->] (h2)	-- (h3);
						\draw [->] (h3)	-- (h4);
					
					\end{tikzpicture}
				\end{center}
			\end{overlayarea}
		\end{column}
		\begin{column}{0.6\textwidth}
			
			
			\begin{itemize}
				\justifying
				\item<1-> This property $(X_i \independent X_{1}^{i-2} | X_{i-1})$ is called the Markov property
				\item<2-> And the resulting chain $X_1, X_2, \dots, X_k$ is called a Markov chain
				\item<3-> Further, since we are considering discrete time steps, this is called a discrete time Markov Chain
				\item<4-> Further, since $X_i$'s take on discrete values this is called a discrete time discrete space Markov Chain
				\item<5-> Okay, but why are we interested in Markov chains? (we will get there soon! for now let us just focus on these definitions)
			\end{itemize}
		\end{column}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%55

\begin{frame}
	\begin{columns}
				
		\begin{column}{0.4\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\vspace{0.5cm}
				\tikzstyle{neuronv}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=orange!30, draw=red!50]
				\tikzstyle{neuronh}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=blue!20, draw=blue!60]
				\tikzstyle{stateTransition}=[thick]
				\tikzstyle{learned}=[text=black]
				\begin{center}
					\begin{tikzpicture}[scale=1.9]
						\node (h1)[neuronh] at (0, 1.5) {$X_1$};
						\node (h2)[neuronh] at (0.7, 1.5) {$X_2$};
						\node (h3)[] at (1.4, 1.5) {$\cdots$};
						\node (h4)[neuronh] at (2.1, 1.5) {$X_k$};
									
									
						\draw [->] (h1)	-- (h2);
						\draw [->] (h2)	-- (h3);
						\draw [->] (h3)	-- (h4);
					
					\end{tikzpicture}
				\end{center}
				\begin{itemize}
					\justifying
					\item Recall that each $X_i \in \{0,1\}^n$
				\end{itemize}
				\onslide<5->{
					\begin{table}[]
					\centering
					\scalebox{0.65}
					{
						\begin{tabular}{|l| c| c |}
							\hline
         						$X_{i-1}$ & $X_{i-2}$ & $T_{ab}$\\
         						\hline
         						1 & 1& 0.05\\
         						%\hline
         						1 & 2 & 0.06\\
         						$\vdots$ & $\vdots$ & $\vdots$\\
         						1 & $l$ & 0.02\\
         						\hline		
         						2 & 1& 0.03\\
         						%\hline
         						2 & 2 & 0.07\\
         						$\vdots$ & $\vdots$ & $\vdots$\\
         						2 & $l$ & 0.01\\
         						\hline	
         						$\vdots$ & $\vdots$ & $\vdots$\\
         						\hline		
         						$l$ & 1& 0.1\\
         						%\hline
         						$l$ & 2 & 0.09\\
         						$\vdots$ & $\vdots$ & $\vdots$\\
         						$l$ & $l$ & 0.21\\
         						\hline	
						\end{tabular}
					}
		
				\end{table}
				}
			\end{overlayarea}
		\end{column}
		\begin{column}{0.6\textwidth}
			
			
			\begin{itemize}
				\justifying
				\item<1-> Let us delve a bit deeper into Markov Chains and define a few more quantities
				\item<2-> Let us assume $2^n = l$ (\textit{i.e.}, $X_i$ can take $l$ values)
				\item<3-> How many values do we need to specify the distribution 
					\begin{align*}
						P(X_i = x_i | X_{i-1} = x_{i-1}) ?\hspace{1em} \onslide<4->{(l^2)}
					\end{align*}
					
				\item<5-> We can represent this as a matrix $T \in l\times l$ where the entry $T_{a,b}$ of the matrix denotes the probability of transitioning to state $b$ from state $a$ (\textit{i.e.}, $P(X_i = b | X_{i-1} = a)$)
				\item<6-> The matrix $T$ is called the transition matrix 
			\end{itemize}
		\end{column}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%55

\begin{frame}
	\begin{columns}
				
		\begin{column}{0.4\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\vspace{1cm}
				\tikzstyle{neuronv}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=orange!30, draw=red!50]
				\tikzstyle{neuronh}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=blue!20, draw=blue!60]
				\tikzstyle{stateTransition}=[thick]
				\tikzstyle{learned}=[text=black]
				\begin{center}
					\begin{tikzpicture}[scale=1.9]
						\node (h1)[neuronh] at (0, 1.5) {$X_1$};
						\node (h2)[neuronh] at (0.7, 1.5) {$X_2$};
						\node (h3)[] at (1.4, 1.5) {$\cdots$};
						\node (h4)[neuronh] at (2.1, 1.5) {$X_k$};
									
									
						\draw [->] (h1)	-- (h2);
						\draw [->] (h2)	-- (h3);
						\draw [->] (h3)	-- (h4);
					
					\end{tikzpicture}
				\end{center}
				%\vspace{0.5cm}
				\begin{table}[]
					\centering
					\scalebox{0.65}
					{
						\begin{tabular}{|l| c| c |}
							\hline
         						$X_{i-1}$ & $X_{i-2}$ & $T_{ab}$\\
         						\hline
         						1 & 1& 0.05\\
         						%\hline
         						1 & 2 & 0.06\\
         						$\vdots$ & $\vdots$ & $\vdots$\\
         						1 & $l$ & 0.02\\
         						\hline		
         						2 & 1& 0.03\\
         						%\hline
         						2 & 2 & 0.07\\
         						$\vdots$ & $\vdots$ & $\vdots$\\
         						2 & $l$ & 0.01\\
         						\hline	
         						$\vdots$ & $\vdots$ & $\vdots$\\
         						\hline		
         						$l$ & 1& 0.1\\
         						%\hline
         						$l$ & 2 & 0.09\\
         						$\vdots$ & $\vdots$ & $\vdots$\\
         						$l$ & $l$ & 0.21\\
         						\hline	
						\end{tabular}
					}
		
				\end{table}
			\end{overlayarea}
		\end{column}
		\begin{column}{0.6\textwidth}
			
			
			\begin{itemize}
				\justifying
				\item<1-> We need to define this transition matrix $T_{ab}$, \textit{i.e.}, 
				\begin{align*}
					P(X_i = b | X_{i-1}=a) ~~\forall a, b ~~\forall i
				\end{align*}

				\item<2-> Why do we need to define this $\forall i~$? \onslide<3->{Well, because this transition probabilities may be different for different time steps}
				\item<4-> For example, the transition in the number of customers may be different from Friday to Saturday (weekend) as compared to from Sunday to Monday(weekday)
				\item<5-> Thus, for a Markov chain $X_1, X_2, \dots, X_k$ we will have $k$ such transition matrices $T_1, T_2, \dots, T_k$
			\end{itemize}
		\end{column}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%55

\begin{frame}
	\begin{columns}
				
		\begin{column}{0.4\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\vspace{1cm}
				\tikzstyle{neuronv}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=orange!30, draw=red!50]
				\tikzstyle{neuronh}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=blue!20, draw=blue!60]
				\tikzstyle{stateTransition}=[thick]
				\tikzstyle{learned}=[text=black]
				\begin{center}
					\begin{tikzpicture}[scale=1.9]
						\node (h1)[neuronh] at (0, 1.5) {$X_1$};
						\node (h2)[neuronh] at (0.7, 1.5) {$X_2$};
						\node (h3)[] at (1.4, 1.5) {$\cdots$};
						\node (h4)[neuronh] at (2.1, 1.5) {$X_k$};
									
									
						\draw [->] (h1)	-- (h2);
						\draw [->] (h2)	-- (h3);
						\draw [->] (h3)	-- (h4);
					
					\end{tikzpicture}
				\end{center}
				%\vspace{0.5cm}
				\begin{table}[]
					\centering
					\scalebox{0.65}
					{
						\begin{tabular}{|l| c| c |}
							\hline
         						$X_{i-1}$ & $X_{i-2}$ & $T_{ab}$\\
         						\hline
         						1 & 1& 0.05\\
         						%\hline
         						1 & 2 & 0.06\\
         						$\vdots$ & $\vdots$ & $\vdots$\\
         						1 & $l$ & 0.02\\
         						\hline		
         						2 & 1& 0.03\\
         						%\hline
         						2 & 2 & 0.07\\
         						$\vdots$ & $\vdots$ & $\vdots$\\
         						2 & $l$ & 0.01\\
         						\hline	
         						$\vdots$ & $\vdots$ & $\vdots$\\
         						\hline		
         						$l$ & 1& 0.1\\
         						%\hline
         						$l$ & 2 & 0.09\\
         						$\vdots$ & $\vdots$ & $\vdots$\\
         						$l$ & $l$ & 0.21\\
         						\hline	
						\end{tabular}
					}
		
				\end{table}
			\end{overlayarea}
		\end{column}
		\begin{column}{0.6\textwidth}
			
			
			\begin{itemize}
				\justifying
				\item<1-> However, for this discussion we will assume that the Markov chain is time homogeneous
				\item<2-> What does that mean? It means that 
					\begin{align*}
						T_1 = T_2 = \dots = T_k = T
					\end{align*}
				\item<3-> In other words 
					\begin{align*}
						P(X_i = b | X_{i-1}=a) = T_{ab} \hspace{1em} \forall a,b \hspace{1em} \forall i
					\end{align*}
				\item<4-> The transition matrix does not depend on the time $i$ and hence such a Markov Chain is called \textit{time homogeneous}
			\end{itemize}
		\end{column}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%55

\begin{frame}
	\begin{columns}
				
		\begin{column}{0.4\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\vspace{3.5cm}
				\tikzstyle{neuronv}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=orange!30, draw=red!50]
				\tikzstyle{neuronh}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=blue!20, draw=blue!60]
				\tikzstyle{stateTransition}=[thick]
				\tikzstyle{learned}=[text=black]
				\begin{center}
					\begin{tikzpicture}[scale=1.9]
						\node (h1)[neuronh] at (0, 1.5) {$X_1$};
						\node (h2)[neuronh] at (0.7, 1.5) {$X_2$};
						\node (h3)[] at (1.4, 1.5) {$\cdots$};
						\node (h4)[neuronh] at (2.1, 1.5) {$X_k$};
									
									
						\draw [->] (h1)	-- (h2);
						\draw [->] (h2)	-- (h3);
						\draw [->] (h3)	-- (h4);
					
					\end{tikzpicture}
				\end{center}
			\end{overlayarea}
		\end{column}
		\begin{column}{0.6\textwidth}
			
			
			\begin{itemize}
				\justifying
				\item<1-> Now suppose the starting distribution at time step 0 is given by $\mu^{0})$
				\item<2-> Just to be clear $\mu^{0}$ is a $2^n$ dimensional vector such that 
					\vspace{-1em}
					\begin{align*}
						\mu^{0}_a = P(X_0 = a)
					\end{align*}
				\item<3-> $\mu^{0}_a$ is the probability that the random variable takes on the value $a$ among all the possible $2^n$ values
				\item<4-> Given $\mu^{0}$ and $T$ how will you compute $\mu^{k}$ where 
					\vspace{-1em}
					\begin{align*}
						\mu^{k}_a = P(X_k = a)
					\end{align*}
				\item<5-> $\mu^{k}$ is again a $2^n$ dimensional vector whose $a^{th}$ entry tells us the probability that $X_k$ will take on the value $a$ among all the possible $2^n$ values	
			\end{itemize}
		\end{column}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
	\begin{columns}
				
		\begin{column}{0.4\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\vspace{1.5cm}
				\tikzstyle{neuronv}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=orange!30, draw=red!50]
				\tikzstyle{neuronh}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=blue!20, draw=blue!60]
				\tikzstyle{stateTransition}=[thick]
				\tikzstyle{learned}=[text=black]
				\begin{center}
					\begin{tikzpicture}[scale=1.9]
						\node (h1)[neuronh] at (0, 1.5) {};
						\node (h2)[neuronh] at (0.9, 1.5) {};
						
						\node (d1)[] at (0, 2.1) {$\vdots$};
						\node (d2)[] at (0.9, 2.1) {$\vdots$};
						
						\node (h3)[neuronh] at (0, 2.7) {};
						\node (h4)[neuronh] at (0.9, 2.7) {b};
						
						\node (h5)[neuronh] at (0, 3.3) {};
						\node (h6)[neuronh] at (0.9, 3.3) {};
									
						\onslide<2->{			
							\draw [->] (h1)	-- (h4);
							\draw [->] (h3)	-- (h4);
							\draw [->] (h5)	-- (h4);
						}
						
						\node[learned,above=0.1cm of h5] (ha1) {$X_0$};
						\node[learned,above=0.1cm of h6] (ha2) {$X_1$};
						
						\node[learned,left=0.1cm of h1] (hl1) {$l$};
						\node[learned,left=0.1cm of h3] (hl2) {$2$};
						\node[learned,left=0.1cm of h5] (hl3) {$1$};
					
					\end{tikzpicture}
				\end{center}
			\end{overlayarea}
		\end{column}
		\begin{column}{0.6\textwidth}
			
			
			\begin{itemize}
				\justifying
				\item<1-> Let us consider $P(X_1 = b)$
					\onslide<2->{
						\begin{align*}
							P(X_1 = b) = \sum_{a} P(X_0 = a, X_1 = b)
						\end{align*}
					}
				\item<3-> The above sum essentially captures all the paths of reaching $X_1 = b$ irrespective of the value of $X_0$ 
				\begin{align*}
					\onslide<4->{P(X_1 = b) &= \sum_{a} P(X_0 = a, X_1 = b)}\\
					\onslide<5->{&= \sum_{a} P(X_0 = a) P(X_1 = b|X_0 = a)}\\
					\onslide<6->{&= \sum_a \mu^{0}_a T_{ab}}
				\end{align*}
			\end{itemize}
		\end{column}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
	\begin{columns}
				
		\begin{column}{0.5\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\tikzstyle{neuronv}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=orange!30, draw=red!50]
				\tikzstyle{neuronh}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=blue!20, draw=blue!60]
				\tikzstyle{stateTransition}=[thick]
				\tikzstyle{learned}=[text=black]
				\begin{center}
				\onslide<2->{
					\begin{tikzpicture}[scale=1.9]
						\node (h1)[neuronh] at (0, 1.5) {3};
						\node (h2)[neuronh] at (1.5, 1.5) {3};
						
						\node (h3)[neuronh] at (0, 2.5) {2};
						\node (h4)[neuronh] at (1.5, 2.5) {2};
						
						\node (h5)[neuronh] at (0, 3.5) {1};
						\node (h6)[neuronh] at (1.5, 3.5) {1};
									
									
						\draw [->] (h1)	-- (h2) node[near start, fill=white, scale=0.5] {0.4};
						\draw [->] (h1)	-- (h4) node[near start, fill=white, scale=0.5] {0.2};
						\draw [->] (h1)	-- (h6) node[near start, fill=white, scale=0.5] {0.4};
						
						\draw [->] (h3)	-- (h2) node[near start, fill=white, scale=0.5] {0.1};
						\draw [->] (h3)	-- (h4) node[near start, fill=white, scale=0.5] {0.6};
						\draw [->] (h3)	-- (h6) node[near start, fill=white, scale=0.5] {0.3};
						
						\draw [->] (h5)	-- (h2) node[near start, fill=white, scale=0.5] {0.3};
						\draw [->] (h5)	-- (h4) node[near start, fill=white, scale=0.5] {0.5};
						\draw [->] (h5)	-- (h6) node[near start, fill=white, scale=0.5] {0.2};
						
						\node[learned,above=0.1cm of h5] (ha1) {$X_0$};
						\node[learned,above=0.1cm of h6] (ha2) {$X_1$};
						
						\node[learned,below=0.08cm of h1, scale=0.7] (hl1) {$0.3$};
						\node[learned,below=0.08cm of h3, scale=0.7] (hl2) {$0.4$};
						\node[learned,below=0.08cm of h5, scale=0.7] (hl3) {$0.3$};
					
					\end{tikzpicture}
					}
					\vspace{-0.5cm}
					\begin{align*}
					\onslide<4->{
					\mu^{0} T &= 
						\begin{bmatrix}
							\alert<5->{0.3} & \alert<5->{0.4} & \alert<5->{0.3} \\
						\end{bmatrix}
						\begin{bmatrix}
							0.2 & \alert<5->{0.5} & 0.3\\
							0.3 & \alert<5->{0.6} & 0.1\\
							0.4 & \alert<5->{0.2} & 0.4\\
						\end{bmatrix}\\
					}
					\onslide<5->{
						&=
						\begin{bmatrix}
							0.3 & \alert{0.45} & 0.25 \\
						\end{bmatrix}
					}	
					\end{align*}
				\end{center}
			\end{overlayarea}

		\end{column}
		\begin{column}{0.5\textwidth}
			
			
			\begin{itemize}
				\justifying
				\item<1-> Let us see if there is a more compact way of writing the distribution $P(X_1)$ (\textit{i.e.}, of specifying $P(X_1 = b) ~\forall b$)
				\item<2-> Let us consider a simple case when $l = 3$ (as opposed to $2^n$)
				\item<3-> Thus, $\mu^{0} \in R^3$ and $T \in R^{3 \times 3}$
				\item<4-> What does the product $\mu^{0}T$ give us ?
				\item<5-> It gives us the distribution $\mu_{1}$! (the $b^{th}$ entry of this vector is $\sum_a \mu^{0}_a T_{ab}$ which is $P(X_1 = b)$)
			\end{itemize}
		\end{column}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
	\begin{columns}
				
		\begin{column}{0.35\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\vspace{1.5cm}
				\tikzstyle{neuronv}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=orange!30, draw=red!50]
				\tikzstyle{neuronh}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=blue!20, draw=blue!60]
				\tikzstyle{stateTransition}=[thick]
				\tikzstyle{learned}=[text=black]
				\begin{center}
					\begin{tikzpicture}[scale=1.9]
						\node (h1)[neuronh] at (0, 1.5) {};
						\node (h2)[neuronh] at (0.9, 1.5) {};
						\node (h3)[neuronh] at (1.8, 1.5) {};
						
						\node (d1)[] at (0, 2.1) {$\vdots$};
						\node (d2)[] at (0.9, 2.1) {$\vdots$};
						\node (d3)[] at (1.8, 2.1) {$\vdots$};
						
						\node (h4)[neuronh] at (0, 2.7) {};
						\node (h5)[neuronh] at (0.9, 2.7) {};
						\node (h6)[neuronh] at (1.8, 2.7) {b};
						
						\node (h7)[neuronh] at (0, 3.3) {};
						\node (h8)[neuronh] at (0.9, 3.3) {};
						\node (h9)[neuronh] at (1.8, 3.3) {};
									
						\onslide<2->{			
							\draw [->] (h2)	-- (h6);
							\draw [->] (h5)	-- (h6);
							\draw [->] (h8)	-- (h6);
						}
						
						\node[learned,above=0.1cm of h7] (ha1) {$X_0$};
						\node[learned,above=0.1cm of h8] (ha2) {$X_1$};
						\node[learned,above=0.1cm of h9] (ha3) {$X_2$};
						
						\node[learned,left=0.1cm of h1] (hl1) {$l$};
						\node[learned,left=0.1cm of h4] (hl2) {$2$};
						\node[learned,left=0.1cm of h7] (hl3) {$1$};
					
					\end{tikzpicture}
				\end{center}
			\end{overlayarea}

		\end{column}
		\begin{column}{0.65\textwidth}
			
			
			\begin{itemize}
				\justifying
				\item<1-> Let us consider $P(X_2 = b)$
					\vspace{-1em}
					\begin{align*}
						\onslide<2->{P(X_2 = b) = \sum_{a} P(X_1 = a, X_2 = b)}
					\end{align*}
				\item<3-> \vspace{-1em} The above sum essentially captures all the paths of reaching $X_2 = b$ irrespective of the value of $X_1$
				\begin{align*}
					\onslide<4->{P(X_2 = b) &= \sum_{a} P(X_1 = a, X_2 = b)}\\
					\onslide<5->{&= \sum_{a} P(X_1 = a) P(X_2 = b|X_1 = a)}\\
					\onslide<6->{&= \sum_a \mu^{1}_a T_{ab}}
				\end{align*}
			\end{itemize}
		\end{column}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
	\begin{columns}
				
		\begin{column}{0.4\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\vspace{1.5cm}
				\tikzstyle{neuronv}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=orange!30, draw=red!50]
				\tikzstyle{neuronh}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=blue!20, draw=blue!60]
				\tikzstyle{stateTransition}=[thick]
				\tikzstyle{learned}=[text=black]
				\begin{center}
					\begin{tikzpicture}[scale=1.9]
						\node (h1)[neuronh] at (0, 1.5) {};
						\node (h2)[neuronh] at (0.9, 1.5) {};
						\node (h3)[neuronh] at (1.8, 1.5) {};
						
						\node (d1)[] at (0, 2.1) {$\vdots$};
						\node (d2)[] at (0.9, 2.1) {$\vdots$};
						\node (d3)[] at (1.8, 2.1) {$\vdots$};
						
						\node (h4)[neuronh] at (0, 2.7) {};
						\node (h5)[neuronh] at (0.9, 2.7) {};
						\node (h6)[neuronh] at (1.8, 2.7) {b};
						
						\node (h7)[neuronh] at (0, 3.3) {};
						\node (h8)[neuronh] at (0.9, 3.3) {};
						\node (h9)[neuronh] at (1.8, 3.3) {};
									
			
						\draw [->] (h2)	-- (h6);
						\draw [->] (h5)	-- (h6);
						\draw [->] (h8)	-- (h6);
						
						\node[learned,above=0.1cm of h7] (ha1) {$X_0$};
						\node[learned,above=0.1cm of h8] (ha2) {$X_1$};
						\node[learned,above=0.1cm of h9] (ha3) {$X_2$};
						
						\node[learned,left=0.1cm of h1] (hl1) {$l$};
						\node[learned,left=0.1cm of h4] (hl2) {$2$};
						\node[learned,left=0.1cm of h7] (hl3) {$1$};
					
					\end{tikzpicture}
				\end{center}
			\end{overlayarea}
		\end{column}
		\begin{column}{0.6\textwidth}
			
			
			\begin{itemize}
				\justifying
				\item<1-> Once again we can write $P(X_2)$ compactly as
				\begin{align*}
					\onslide<2->{P(X_2) =  \mu^{1}T}
					       \onslide<3->{=  (\mu^{0}T)T}
					       \onslide<4->{= \mu^{0}T^2}
				\end{align*}
				\item<5-> In general, 
					\begin{align*}
						P(X_k) = \onslide<6->{\mu^{0}T^k}
					\end{align*}
				\item<7-> Thus the distribution at any time step can be computed by finding the appropriate element from the following series
					\begin{align*}
						\mu^{0}T^1, \mu^{0}T^2, \mu^{0}T^3, \dots, \mu^{0}T^k, \dots
					\end{align*}
				
				\item<8-> Note that this is still computationally expensive because it involves a product of $\mu^{0} (2^n)$ and $T^{k} (2^n \times 2^n)$ (but later on we will see that we do not need this full product)
			\end{itemize}
		\end{column}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
	\begin{columns}
				
		\begin{column}{0.4\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\vspace{1.5cm}
				\tikzstyle{neuronv}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=orange!30, draw=red!50]
				\tikzstyle{neuronh}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=blue!20, draw=blue!60]
				\tikzstyle{stateTransition}=[thick]
				\tikzstyle{learned}=[text=black]
				\begin{center}
					\begin{tikzpicture}[scale=1.9]
						\node (h1)[neuronh] at (0, 1.5) {};
						\node (h2)[neuronh] at (0.9, 1.5) {};
						\node (h3)[neuronh] at (1.8, 1.5) {};
						
						\node (d1)[] at (0, 2.1) {$\vdots$};
						\node (d2)[] at (0.9, 2.1) {$\vdots$};
						\node (d3)[] at (1.8, 2.1) {$\vdots$};
						
						\node (h4)[neuronh] at (0, 2.7) {};
						\node (h5)[neuronh] at (0.9, 2.7) {};
						\node (h6)[neuronh] at (1.8, 2.7) {b};
						
						\node (h7)[neuronh] at (0, 3.3) {};
						\node (h8)[neuronh] at (0.9, 3.3) {};
						\node (h9)[neuronh] at (1.8, 3.3) {};
									
			
						\draw [->] (h2)	-- (h6);
						\draw [->] (h5)	-- (h6);
						\draw [->] (h8)	-- (h6);
						
						\node[learned,above=0.1cm of h7] (ha1) {$X_0$};
						\node[learned,above=0.1cm of h8] (ha2) {$X_1$};
						\node[learned,above=0.1cm of h9] (ha3) {$X_2$};
						
						\node[learned,left=0.1cm of h1] (hl1) {$l$};
						\node[learned,left=0.1cm of h4] (hl2) {$2$};
						\node[learned,left=0.1cm of h7] (hl3) {$1$};
					
					\end{tikzpicture}
				\end{center}
			\end{overlayarea}
		\end{column}
		\begin{column}{0.6\textwidth}
			
			
			\begin{itemize}
				\justifying
				\item<1-> If at a certain time step $t$, $\mu^{t}$ reaches a distribution $\pi$ such that $\pi T = \pi$
				\item<2-> Then for all subsequent time steps
					\begin{align*}
						\mu^{j} = \pi (j \geq t)
					\end{align*}
				\item<3-> $\pi$ is then called the stationary distribution of the Markov chain
				\item<4->  $X_{t}, X_{t+1}, X_{t+2}, \dots$ will all follow the same distribution $\pi$
				\item<5-> In other words, if we have $X_{t} = x_t, X_{t+1} = x_{t+1}, X_{t+2} = x_{t+2}$ and so on then we can think of $x_t, x_{t+1}, x_{t+2}$ as samples drawn from the same distribution $\pi$ (this is a crucial property and we will return back to it soon)
			\end{itemize}
		\end{column}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
	\begin{columns}
				
		\begin{column}{0.4\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\vspace{1.5cm}
				\tikzstyle{neuronv}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=orange!30, draw=red!50]
				\tikzstyle{neuronh}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=blue!20, draw=blue!60]
				\tikzstyle{stateTransition}=[thick]
				\tikzstyle{learned}=[text=black]
				\begin{center}
					\begin{tikzpicture}[scale=1.9]
						\node (h1)[neuronh] at (0, 1.5) {};
						\node (h2)[neuronh] at (0.9, 1.5) {};
						\node (h3)[neuronh] at (1.8, 1.5) {};
						
						\node (d1)[] at (0, 2.1) {$\vdots$};
						\node (d2)[] at (0.9, 2.1) {$\vdots$};
						\node (d3)[] at (1.8, 2.1) {$\vdots$};
						
						\node (h4)[neuronh] at (0, 2.7) {};
						\node (h5)[neuronh] at (0.9, 2.7) {};
						\node (h6)[neuronh] at (1.8, 2.7) {b};
						
						\node (h7)[neuronh] at (0, 3.3) {};
						\node (h8)[neuronh] at (0.9, 3.3) {};
						\node (h9)[neuronh] at (1.8, 3.3) {};
									
			
						\draw [->] (h2)	-- (h6);
						\draw [->] (h5)	-- (h6);
						\draw [->] (h8)	-- (h6);
						
						\node[learned,above=0.1cm of h7] (ha1) {$X_0$};
						\node[learned,above=0.1cm of h8] (ha2) {$X_1$};
						\node[learned,above=0.1cm of h9] (ha3) {$X_2$};
						
						\node[learned,left=0.1cm of h1] (hl1) {$l$};
						\node[learned,left=0.1cm of h4] (hl2) {$2$};
						\node[learned,left=0.1cm of h7] (hl3) {$1$};
					
					\end{tikzpicture}
				\end{center}
			\end{overlayarea}
		\end{column}
		\begin{column}{0.6\textwidth}
			
			
			\begin{itemize}
				\justifying
				\item<1-> \textbf{Important:} If we run a Markov Chain for a large number of time steps then after a point we start getting samples $x_t, x_{t+1}, x_{t+2}, \dots$ which are essentially being drawn from the stationary distribution (\textbf{Spoiler Alert:} one of our goals was to draw samples from a very complex distribution)
				\item<2-> What do we mean by run a Markov Chain for a large number of time steps ?
				\item<3-> It means we start drawing a sample $X_0 \sim \mu^{0}$ and then continue drawing samples 
				\begin{align*}
					X_1 \sim \mu^{0}T,\hspace{0.5em} X_2 \sim \mu^{0}T^2,\hspace{0.5em} X_3 \sim \mu^{0}T^3, \dots,\\\dots,  X_t \sim \pi,\hspace{0.5em} X_{t+1} \sim \pi,\hspace{0.5em} X_{t+2} \sim \pi \dots
				\end{align*}
			\end{itemize}
		\end{column}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
	\begin{columns}
				
		\begin{column}{0.4\textwidth}
			\begin{overlayarea}{\textwidth}{\textheight}
				\vspace{1.5cm}
				\tikzstyle{neuronv}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=orange!30, draw=red!50]
				\tikzstyle{neuronh}=[circle,minimum size=20pt,inner sep=0pt, thick, fill=blue!20, draw=blue!60]
				\tikzstyle{stateTransition}=[thick]
				\tikzstyle{learned}=[text=black]
				\begin{center}
					\begin{tikzpicture}[scale=1.9]
						\node (h1)[neuronh] at (0, 1.5) {};
						\node (h2)[neuronh] at (0.9, 1.5) {};
						\node (h3)[neuronh] at (1.8, 1.5) {};
						
						\node (d1)[] at (0, 2.1) {$\vdots$};
						\node (d2)[] at (0.9, 2.1) {$\vdots$};
						\node (d3)[] at (1.8, 2.1) {$\vdots$};
						
						\node (h4)[neuronh] at (0, 2.7) {};
						\node (h5)[neuronh] at (0.9, 2.7) {};
						\node (h6)[neuronh] at (1.8, 2.7) {b};
						
						\node (h7)[neuronh] at (0, 3.3) {};
						\node (h8)[neuronh] at (0.9, 3.3) {};
						\node (h9)[neuronh] at (1.8, 3.3) {};
									
			
						\draw [->] (h2)	-- (h6);
						\draw [->] (h5)	-- (h6);
						\draw [->] (h8)	-- (h6);
						
						\node[learned,above=0.1cm of h7] (ha1) {$X_0$};
						\node[learned,above=0.1cm of h8] (ha2) {$X_1$};
						\node[learned,above=0.1cm of h9] (ha3) {$X_2$};
						
						\node[learned,left=0.1cm of h1] (hl1) {$l$};
						\node[learned,left=0.1cm of h4] (hl2) {$2$};
						\node[learned,left=0.1cm of h7] (hl3) {$1$};
					
					\end{tikzpicture}
				\end{center}
			\end{overlayarea}
		\end{column}
		\begin{column}{0.6\textwidth}
			
			
			\begin{itemize}
				\justifying
				\item<1-> Is it always easy to draw these samples? \onslide<2->{No}
				\item<3-> $|\mu^{k}| = 2^n$ which means that we need to compute the probability of each of the possible $2^n$ values that $X^{k}$ can take
				\item<4-> In other words the joint distribution $\mu^{k}$ has $2^n$ parameters which is prohibitively large
				\item<5-> I wonder what can I do to reduce the number of parameters in a joint distribution (I hope you already know what to do but we will return back to it later)
			\end{itemize}
		\end{column}
	\end{columns}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
	\begin{block}{The story so far...}
	\begin{itemize}	
		 \item<1-> We have seen what a discrete space, discrete time, time homogeneous Markov Chain is
		\item<2-> We have also defined $\mu^{0}$ (initial distribution), $T$ (transition matrix) and $\pi$ (stationary distribution)
		\item<3-> So far so good! But why do we care about Markov Chains and their properties?
		\item<4->How does this discussion tie back to our goals?
		\item<5-> We will first see an intuitive explanation for how all this ties back to our goals and then get into a more formal discussion
		\end{itemize}
	\end{block}
\end{frame}
