\begin{frame}
	\myheading{Module 14.5: Some Gory Details}
\end{frame}

\begin{frame}
    \begin{overlayarea}{\textwidth}{\textheight}
        \[
        \underbrace{\frac{\partial \mathscr{L}_t(\theta)}{\partial W}}_{\onslide<2->{\in \mathbb{R}^{d \times d}}} =
        \underbrace{\frac{\partial \mathscr{L}_t(\theta)}{\partial s_t}}_{\onslide<3->{\in \mathbb{R}^{1 \times d}}}
        \sum_{k=1}^{t} \underbrace{\frac{\partial s_t}{\partial s_k}}_{\onslide<4->{\in \mathbb{R}^{d \times d}}}
        \underbrace{\frac{\partial^+ s_k}{\partial W}}_{\onslide<5->{\in \mathbb{R}^{d \times d \times d}}}
        \]
        \begin{itemize}
            \item<6-> We know how to compute $\frac{\partial \mathscr{L}_t(\theta)}{\partial s_t}$ (derivative of $\mathscr{L}_t(\theta)$ (scalar)
            w.r.t. last hidden layer (vector)) using backpropagation
            \item<7-> We just saw a formula for $\frac{\partial s_t}{\partial s_k}$ which is the derivative of a vector w.r.t. a vector)
            \item<8-> $\frac{\partial^+ s_k}{\partial W}$ is a tensor $\in \mathbb{R}^{d \times d \times d}$, the
            derivative of a vector $\in \mathbb{R}^d$ w.r.t. a matrix $\in \mathbb{R}^{d \times d}$
            \item<9-> How do we compute $\frac{\partial^+ s_k}{\partial W}$ ? \onslide<6-> {Let us see}
        \end{itemize}
    \end{overlayarea}
\end{frame}

\begin{frame}
    %\begin{overlayarea}{\textwidth}{\textheight}
    \begin{itemize}

        \item<1-> We just look at one element of this $\frac{\partial^+ s_k}{\partial W}$ tensor
        \item<2-> $\frac{\partial^+ s_{kp}}{\partial W_{qr}}$ is the $(p,q,r)$-th element of the
        3d tensor
        \item[]<3->   $a_k = Ws_{k-1} + b$
        \item[]<4->   $s_k = \sigma(a_k)$
        %\onslide<4->{
        %  \begin{flalign*}
        %    \onslide<4->{a_k &= Ws_{k-1} + b} \\
        %    \onslide<5->{s_k &= \sigma(a_k)}
        %  \end{flalign*}
        %}
    \end{itemize}
    %\end{overlayarea}
\end{frame}

\begin{frame}
    \small{
        \begin{columns}
            \column{0.5\textwidth}
            \begin{overlayarea}{\textwidth}{\textheight}
                \begin{align*}
                \onslide<1->{
                    a_k &= W s_{k-1}} \\
                \onslide<2->{
                    \begin{bmatrix}
                    a_{k1} \\
                    a_{k2} \\
                    \vdots \\
                    \textcolor{red}{a_{kp}} \\
                    \vdots \\
                    a_{kd}
                    \end{bmatrix} &=
                    \begin{bmatrix}
                    W_{11} & W_{12} & \dots & W_{1d} \\
                    \\
                    \vdots & \vdots & \vdots & \vdots \\
                    \textcolor{red}{W_{p1}} & \textcolor{red}{W_{p2}} & \textcolor{red}{\dots} & \textcolor{red}{W_{pd}} \\
                    \vdots & \vdots & \vdots & \vdots \\
                    \\
                    \end{bmatrix}
                   \textcolor{red}{ \begin{bmatrix}
                    s_{k-1, 1} \\
                    s_{k-1, 2} \\
                    \vdots \\
                    s_{k-1, p} \\
                    \vdots \\
                    s_{k-1, d}
                    \end{bmatrix}} }\\
                \onslide<3->{
                    a_{kp} &= \sum_{i=1}^{d} W_{pi} s_{k-1,i}} \\
                \onslide<4->{
                    s_{kp} &= \sigma(a_{kp})} \\
                \onslide<5->{
                    \frac{\partial s_{kp}}{\partial W_{qr}} &= \frac{\partial s_{kp}}{\partial a_{kp}} \frac{\partial a_{kp}}{\partial W_{qr}}}\\
                \onslide<6->{
                    &= \sigma'(a_{kp}) \frac{\partial a_{kp}}{\partial W_{qr}}}
                \end{align*}
            \end{overlayarea}

            \column{0.5\textwidth}
            \begin{overlayarea}{\textwidth}{\textheight}
                \begin{align*}
                \onslide<7->{
                    \frac{\partial a_{kp}}{\partial W_{qr}} &= \frac{\partial \sum_{i=1}^d W_{pi} s_{k-1,i}}{\partial W_{qr}}} \\
                \onslide<8->{
                    &= s_{k-1,i}  \quad \text{if} \quad p = q \quad \text{and} \quad i = r} \\
                \onslide<9->{
                    &= 0 \quad \text{otherwise}} \\
                \onslide<10->{
                    \frac{\partial s_{kp}}{\partial W_{qr}} &= \sigma'(a_{kp})s_{k-1, r} \quad \text{if} \quad p = q \quad \text{and} \quad i = r} \\
                \onslide<11->{
                    &= 0 \quad \text{otherwise}}
                \end{align*}
            \end{overlayarea}
        \end{columns}
    }
\end{frame}
