\documentstyle[12pt,fullpage,psfig]{article}

\newcommand{\lecturetitle}[4]{
  \noindent
  CS 294-2, Grouping and Recognition (Prof. Jitendra Malik) \hfill #3\\
  Lecture \##1 (#2) \hfill {\bf DRAFT} Notes by #4\\[-.05in]
  \mbox{}\hrulefill \mbox{}\\}

\begin{document}

% MODIFY THE FOLLOWING TO INCLUDE THE LECTURE #, TOPIC, DATE, and YOUR NAME
\lecturetitle{4}{Expectation Maximization Algorithm} 
{Sept 1st, 1999}{Ren\'e Vidal}

% INCLUDE AN OUTLINE OF THE CLASS TOPICS
\begin{itemize} 
  \item Mixture models with incomplete data 
	\begin{itemize}
	\item Expectation Maximization Algorithm
	\item EM for Gaussian distribution
	\item Application to images and disadvantages of the method
	\end{itemize}
\end{itemize}

% THE NOTES WILL GO HERE

\section{Mixture Models with incomplete data}

In last lecture, we discussed how to estimate the parameters
$\theta_k=(\pi_k, \phi_k), k=1\ldots K$ of the probability
distribution of the mixture model by assuming complete data knowledge,
that is, we assumed we had measurements for both $x$ and $q$. In
this lecture, we will discuss how to estimate the parameters when we
have incomplete data knowledge. We will concentrate in the case where
$q$ is not known.


\subsection{Expectation Maximization Algorithm}
In the complete data case, we computed the parameters of the model by
maximizing the log-likelihood of the joint probability density of
$x$ and $q$. Now $q$ is unobserved, thus we might try to estimate
the marginal density of $x$
\begin{equation}
\sum_{q} P(x,q;\theta) %\qquad {or} \qquad \int P(x,y;\theta) dy
\end{equation}
and then apply the maximum likelihood method. In order to do that,
assume we have a guess $\theta'$ of the value of the parameters, and
consider a series of measurements of $x$, $x_1, \ldots, x_N$. Thus if
we are able to construct a function $Q(\theta,\theta',(x_1,\ldots
x_N))$ that estimates the log-likelihood of $x$ given the guess of the
parameters and the measurements on $x$, then we can find a new
estimate of the parameters that maximizes $Q$. This procedure can be
repeated to find a sequence of better approximations of $\theta$. Such
an iterative algorithm is called Expectation Maximization (EM) and can
be summarized as follows


\begin{enumerate}
\item[E-step:] Construct a function $Q(\theta,\theta',(x_1,\ldots
x_N))$, the so-called
``Expected complete log-likelihood'' based on the previous guess $\theta'$
\item[M-step:] Find $\theta$ that maximizes $Q$
\item[Iterate] Repeat E and M steps until the difference between
subsequent estimates is small
\end{enumerate}

For the E-step, we can choose $Q$ as the expected value of the
log-likelihood of $x$ given the measurements, that is:

\begin{eqnarray*}
Q(\theta,\theta',(x_1,\ldots,x_N)) &=& E \left [ \sum_{i=1}^N
\sum_{k=1}^K \log \left (P(q_i^k,x_i;\theta)\right) | \:
x=(x_1,\ldots, x_N)\right] \\
&=&  \sum_{i=1}^N \sum_{k=1}^K \sum_{q} \log \left
(P(q_i^k,x_i;\theta)\right) P(q_i^k | x_i;\theta') \\
&=&  \sum_{i=1}^N \sum_{k=1}^K \sum_{q} q_i^k \log \left
(\pi_k f_k(x_i,\phi_k)\right) P(q_i^k | x_i;\theta') \\
&=&  \sum_{i=1}^N \sum_{k=1}^K  <q_i^k> \log \left
(\pi_k f_k(x_i,\phi_k)\right)  
\end{eqnarray*}
where
\begin{eqnarray*}
<q_i^k> &=&  \sum_{q} q_i^k  P(q_i^k | x_i;\theta') \\
&=& \frac{\sum_{q} P(x_i | q_i^k ;\theta') P(q_i^k)
q_i^k}{P(x_i;\theta')}\\
&=& \frac{\pi_k' f_k (x_i;\phi_k')}{\sum_{k=1}^K
P(x_i|q_i^k;\theta')P(q_i^k)}\\
&=& \frac{\pi_k' f_k (x_i;\phi_k')}{\sum_{k=1}^K
\pi_k' f_k (x_i;\phi_k') }
\end{eqnarray*}

\subsection{EM for Gaussian Distribution}
If the distribution of the images $x$ conditioned on $q$ is Gaussian, that is:
\begin{equation}
f_k = \frac{1}{\sqrt{2\pi} \: \sigma_k} \: \exp
\left(-\frac{(x-\mu_k)^2}{2\sigma_k^2}\right),
\end{equation}
then we have to optimize
\begin{eqnarray*}
{\cal L } = \sum_{i=1}^N \sum_{k=1}^K  <q_i^k> \log \left
(\pi_k f_k(x_i,\phi_k)\right)  + \lambda(1-\sum_{k=1}^K \pi_k)
\end{eqnarray*}
where $\lambda$ is the Lagrange multiplier.

Taking derivatives with respect to $\pi_k$ and
$\phi_k=(\mu_k,\sigma_k)$, and recalling that $\sum_k <q_i^k>=1$, we
get
\begin{eqnarray*}
\frac{\partial Q}{\partial \pi_k} &=& \sum_{i=1}^N \frac{<q_i^k>}{\pi_k}
- \lambda = 0 \qquad \qquad \qquad \Rightarrow \: \widehat{\pi_k} =
\frac{\sum_{i=1}^N <q_i^k>}{N} \\ 
\frac{\partial Q}{\partial \mu_k} &=& \sum_{i=1}^N \frac{<q_i^k>
(x_i-\mu_k)}{\sigma_k^2} = 0 \qquad \quad \: \: \: \Rightarrow \:
\widehat{\mu_k} =
\frac{\sum_{i=1}^N <q_i^k> x_i}{\sum_{i=1}^N <q_i^k>} \\
\frac{\partial Q}{\partial \sigma_k} &=& \sum_{i=1}^N \frac{<q_i^k>
\left [ (x_i-\mu_k)^2-\sigma_k^2\right ]}{\sigma_k^3} = 0 \: \:
\Rightarrow \: \widehat{\sigma_k}^2 = \frac{\sum_{i=1}^N <q_i^k>
(x_i-\widehat{\mu_k})^2}{\sum_{i=1}^N <q_i^k>} \\
\end{eqnarray*}
This formulae give us new estimation for $\theta$ given the current
estimation $\theta'$ and the measurements $(x_1, \ldots x_N)$. Recall
that $<q_i^k>$ is a function of  $\theta'$ and $(x_1, \ldots x_N)$.

More generally, we might assume that the distribution is a
Multivariate Gaussian. In this case the joint density is given by:
\begin{eqnarray*}
f({\bf x}; \mu,\Sigma) = \frac{1}{(2\pi)^{d/2}\det(\Sigma)^{1/2}} \: \exp
\left(-\frac{({\bf x}-\mu)^T\Sigma^{-1}({\bf x}-\mu)}{2}\right),
\end{eqnarray*}
where $\bf x$ is a vector in $R^d$, $\mu \in R^d$ is the mean, and
$\Sigma\in R^{d\times d}$ is the covariance matrix. 

\subsection{Application to images}
Based on the EM philosophy, the following scheme (see Figure
\ref{em1}) can be used for image segmentation
\begin{itemize}
\item Assume we know to which class each pixel belongs to (pixel
classes) and find class characteristics
\item Assume we know class characteristics and find pixel classes
\item Iterate
\end{itemize}

% EXAMPLE OF A PostScript FIGURE
\begin{figure}[h]
\centerline{\psfig{file=em1.eps,height=3in,angle=90.5}}
\caption{Grouping: Expectation-Maximization}
\label{em1}
\end{figure}

\newpage

Figure \ref{em2} shows an example of the application of EM to image
segmentation

\begin{figure}[h]
\centerline{\psfig{file=em2.eps,height=3in,angle=90.5}}
\caption{Using EM to group image regions}
\label{em2}
\end{figure}



\subsection{Disadvantages of the method}
\begin{itemize}
\item Can converge to a local minima
\item Sensitive to initialization
\item We do not know how to choose $K$.
\end{itemize}




\end{document}

