\documentclass[11pt]{article}

\usepackage{lecture}
\usepackage{amsmath}
\usepackage{verbatim} %To be removed. Needed for Comment environment.

\begin{document}

\lecture{17}{3/20/2013}{Randomness Extractors}{Mahnaz Akbari}

\draft % put this here only if you want to indicate this is a draft

This lecture is about randomness extractors.
Extractors are functions that map samples from a non-uniform 
distribution to samples that are close to being uniformly distributed.
The length of the output will in general be smaller than the length of 
the input of the extractor.

The input distribution of the extractor is called the \emph{source}.
A source is a random variable which maps text values to bit strings. 
The goal is to extract randomness from such sources.
\begin{definition}
  Let $(\Omega,p)$ be probability space, that is, $\Omega$ is a finite 
  set and $p\in [0,1]^{\Omega}$ is a vector with $\|p\|_1=1$.
  A \emph{source} $X$ over $\zo^n$ is a $\zo^n$-valued random variable 
  over some space $(\Omega,p)$, that is, a function
  \begin{equation*}
    X \ : \ \Omega \ \rightarrow \ \{0,1\}^{n}\,.
  \end{equation*}
\end{definition}
We recall the definition of an extractor from Lecture~2, where we gave 
a general overview.
\begin{definition}[Extractor] \label{def:unseeded-extractor}
  Let $C$ be a class of sources over $\zo^n$.
  A function $E : \{0,1\}^{n} \to \{0,1\}^{r}$ is a (deterministic) 
  $\epsilon$-extractor for $C$ if,
  for all sources $X\in C$, we have
  \begin{equation*}
    \sdist\Big(U_{r}\;,\;E(X)\Big) \leq \epsilon\,.
  \end{equation*}
\end{definition}
We give an example of a source an how to extract randomness from it.
\begin{example}
Let $X=X_{1}X_{2} \dots X_{n}$ be a random variable on $\zo^n$ so that 
the bits $X_i$ are identically and independently distributed with 
$P(X_{i}=1)= \sigma$.
So the source is not uniform if $\sigma\neq \frac{1}{2}$
To extract a uniform distribution from this source, one can 
partition~$X$ into strings of length two.
Let us consider the first pair of bits, namely $X_{1}$ and $X_{2}$.
Then we observe that the events $X_1X_2 =01$ and $X_1 X_2 = 10$ have 
the same probability:\\

\begin{tabular}{llll}
$X_{1}$ & $X_{2}$ & $\Pr$ & $E$ outputs \\
\hline
0 & 1 & $\sigma(1-\sigma)$ & 1\\
1 & 0 & $\sigma(1-\sigma)$ & 0\\
0 & 0 & $\sigma^2$         & discard\\
1 & 1 & $(1-\sigma)^2$     & discard
\end{tabular}
\\
\\
Since the first two events happen with the same probability, they can 
be used to produce a single unbiased bit.
In case the bits happen to be $00$ or $11$, we can't extract unbiased 
bits directly, and we simply move on to the next pair of bits.
On average, the number of bits that this procedure produces is the 
number of pairs for which one of these two events occurs.
Thus, the expected length of the output of the extractor is 
$\Exp(r)=2\sigma(1-\sigma) \frac{n}{2}$.
\end{example}

The example above leads us to the question of how much randomness is 
contained in a source.
In the extreme case where $X$ is a constant, there is no hope to 
extract any randomness.
On the other hand, if $X$ is a source on $\zo^n$, we cannot expect to 
be able to extract more than $n$ uniform bits.
In the example above, the amount of randomness that we could extract 
was a constant fraction of $n$.

\section{Entropy measures}
\emph{Entropy} measures the amount of randomness contained in a random 
source.
There are various notions of entropy, two of which we will discuss 
now.
\begin{definition}[Shannon Entropy of $X$]
  Let $X$ be a random variable.
  The \emph{Shannon entropy} of~$X$ is the number
  \begin{align*}
    H(X) &\doteq \Exp_{x\sim X}\underbrace{\left(\log 
        \frac{1}{\Pr[X=x]}\right)}_{\text{\upshape``amount of 
        information contained in $x$''}} \\
    &= \sum_{x} \Pr[X=x] \cdot \left(\log \frac{1}{\Pr[X=x]}\right)\,.
  \end{align*}
\end{definition}
One way to look at the part that shows the amount of information in 
the above formula is that it is, intuitively, the number of bits 
required to store the information in~$x$, when the distribution~$X$, 
from which $x$ was sampled, is know to the receiver.
Then the expectation is the average number of bits needed to store a 
sample from $X$.
In other words, given a sample from $X$, how far it can be compressed.
This intuition is made precise in Shannon's noisy-channel coding 
theorem.

If $X$ is uniformly distributed, the entropy of $X$ is equal to the 
dimension of~$X$ and the identity map is a perfect extractor.
Furthermore, the closer the Shannon entropy is to the dimension of 
$X$, the closer $X$ must be to the uniform distribution, so it is 
reasonable to suspect that a high Shannon entropy allows us to extract 
randomness that is close to uniform.
The following example refutes this intuition for the Shannon entropy.
\begin{example}
 Let $X$ be a random variable on $\zo^n$ with
 \begin{equation*}
   \Pr[X=x] =
   \begin{cases}
     \frac{1}{2} & \text{if $x=0^{n}$,}\\
     \frac{1}{2} \cdot \frac{1}{2^{n-1}} & \text{otherwise.}
   \end{cases}
 \end{equation*}
 This random variable is equal to $0^n$ with probability half, and it 
 is uniformly distributed on the rest of the space.
 Then the Shannon entropy is with $H(X)\approx \frac{n}{2}$ fairly 
 close to maximal, but we also have the following property for all 
 function~$E$:
 \begin{equation*}
   \Big|
   \underbrace{\Pr\big(E(X) = E(0^{n})\big)}_{\geq \frac{1}{2}}
   -
   \underbrace{\Pr\big(U_{r} = E(0^{n})\big)}_{2^{-r}}
   \Big|
   \approx
   \frac{1}{2}
  \end{equation*}
  This implies that the statistical distance between~$E(X)$ is close 
  to~$0.5$ and therefore far away from~$0$ unless~$r$ is trivially 
  small.
  This means that there does not exist an extractor for this random 
  variable.
\end{example}
The example above shows that one can define a random variable~$X$ 
which has a high Shannon entropy, but from which no uniform randomness 
can be extracted with an extractor.
Thus, Shannon entropy is insufficient to measure the extractable 
randomness contained in a random variable, for which reason we will 
use a different notion of entropy.
The problem with the above example was that there was an outcome that 
individually occurs with large probability.
The Min-entropy defined below yields an upper bound on the largest 
probability for individual outcomes.
\begin{definition}[Min-entropy of X]
  Let $X$ be a random variable.
  The \emph{Min-entropy} of~$X$ is the number
  \begin{align*}
    H_{\infty}(X) &\doteq \min_x \log \frac{1}{\Pr[X=x]}
    = \log \frac{1}{\max_x \Pr[X=x]}\,.
  \end{align*}
  Equivalently, $H_{\infty}(X)$ is the largest number $k$ such that 
  all outcomes have probability at most $2^{-k}$.
\end{definition}
In the example above, we had $\max_x\{\Pr[X=x]\} = \frac{1}{2}$ and 
thus $H_\infty(X) = 1$, which is very small compared to the maximum 
possible value of~$n$.
Thus, the Min-entropy may be small even though the Shannon entropy is 
large.
The following lemma shows the relationship between two entropies in 
general.
\begin{lemma}[Relationship Between Entropies.]
  Let $X$ be a random variable and let
  $\supp(X) \doteq \{x|\Pr[X=x]>0\}$
  be the \emph{support} of~$X$.
  Then we have
  \begin{align}\label{eq:relationship}
    0\leq H_{\infty}(X) \leq H(X) \leq \log \big|\supp(X)\big|\,.
  \end{align}
\end{lemma}
The last inequality in~\eqref{eq:relationship} means that a sample of 
$X$ can always be stored using at most $\log |\supp(X)|$ bits.
The Shannon-entropy may be smaller than that, which means that, on 
average, fewer bits are needed to store a sample from~$X$.
The Min-entropy can not be larger than that, and it is the number of 
uniform random that can be extracted from samples of~$X$.

\begin{lemma}
  Let $X$ and $Y$ be independent random variables.
  Then both entropies are additive:
  \begin{align*}
    H(X,Y) &= H(X) + H(Y)\,,\\
    H_\infty(X,Y) &= H_\infty(X) + H_\infty(Y)\,.
  \end{align*}
\end{lemma}
The additivity of the Shannon entropy says that, in order to store a 
sample from $X$ and an independent sample from $Y$, we need to store 
both samples individually.
For the min-entropy, the intuition is that the amount of randomness 
extractable from $X$ and $Y$ is equal to the sum of the amount of 
randomness extractable from these random variables individually.

We use random variables $X$ as a source of (possibly non-uniform) 
randomness.
\begin{definition}
A $k$-source is a random variable $X$ with $H_{\infty}(X) \geq k$.
\end{definition}
As we will see later, one can extract roughly $k$ uniform bits from a 
$k$-source.
Let us first look at some examples of $k$-sources.
\begin{example}[Bit-fixing sources]
  A random variable $X$ on $\zo^n$ is a \emph{bit-fixing source} if
  \begin{itemize}
    \item $k$ bits of $X$ are uniformly distributed
    \item $n-k$ bits of $X$ are fixed; these bits are deterministic 
      function of the uniform $k$ bits.
  \end{itemize}
\end{example}
A special case of bit-fixing sources occurs when the fixed bits are 
all fixed to~$0$.
\begin{example}[Flat $k$-sources]
  A random variable~$X$ on $\zo^n$ is a flat $k$-source if there is a 
  set $S \subseteq \{0,1\}^{n}$ with $|S|=2^{k}$ such that
  \begin{align*}
    \Pr[X=x]
    &=
    \begin{cases}
      \frac{1}{2^{k}} & \text{if $x\in S$,}\\
      0 & \text{otherwise.}
    \end{cases}
  \end{align*}
  We also write~$X_S$ for the flat source.
\end{example}
In both of the cases above, the min-entropy is $H_{\infty}(X) = k$.
Actually, \emph{every} $k$-source is a convex combination of flat 
$k$-sources.
\begin{lemma}\label{lemma:convex}
  Every $k$-source~$X$ is a convex combination of flat $k$-sources, 
  that is,
  \begin{align*}
    \Pr[X=x] =  \sum_{S} c_{S} \cdot \Pr[X_{S}=x]
  \end{align*}
  holds for some coefficients $c_S$ with $\sum_S c_{S} = 1$ and
  $c_{S} \geq 0$.
\end{lemma}
This lemma will enable us to reduce the extraction from an arbitrary 
$k$-source to the extraction from a flat $k$-source.

\begin{exercise}
  Prove Lemma~\ref{lemma:convex}.
\end{exercise}

\section{Deterministic extractors}

The following lemma shows that every deterministic extractor as 
defined in Definition~\ref{def:unseeded-extractor} is bad for some 
source.
\begin{lemma}
  For all functions $E:\zo^n\to\zo$, there is a flat $(n-1)$-source so 
  that $E(X)$ is a constant function.
\end{lemma}
In particular, every extractor has a source of very high entropy for 
which not even a single bit can be extracted.
\begin{proof}
  Let $b\in\zo$ so that the number of $x\in\zo^n$ such that $E(x)=b$ 
  holds is at least $2^n/2=2^{n-1}$.
  Such $b$ exists by the pigeon hole principle.
  Now let $X$ be the flat source whose support consists of those $x$ 
  for which $E(x)=b$ holds.
  Then $X$ is a flat $(n-1)$-source and $E(X)$ is always equal to $b$.
\end{proof}
On the other hand, the following lemma shows that, for every 
$k$-source, \emph{most} deterministic extractors work.
\begin{lemma}\label{lemma:random}
  For all $k$-sources $X$ on $\zo^n$ and $\epsilon>0$ we have
  \begin{align*}
    \Pr_{E}
    [
    \text{random function $E$ is an $\epsilon$-extractor for~$X$}
    ]
    \geq
    1 - 2^{- \Omega (2^{k} \epsilon^{2})}
  \end{align*}
   where $r=k-2 \log \frac{1}{\epsilon} - O(1)$ and $E : \{0,1\}^{n} \to \{0,1\}^{r}$.
\end{lemma}
The lower bound for the probability is very close to one, which means 
that a random function is an extractor for the flat $k$-source $X$ 
with high probability.

\begin{exercise}
  Prove Lemma~\ref{lemma:random} using the Chernoff bound.
\end{exercise}

\section{Seeded Extractors}

We have seen that deterministic extractors cannot be used for all 
$k$-sources, and fully random extractors are good for all $k$-sources.
With our goal of saving randomness in mind, we want to avoid using 
fully random extractors.
\emph{Seeded extractors} are a compromise between the two extremes.
They assume to have access to a little bit of perfectly uniform 
randomness, but the amount is only logarithmic in the length of the 
source.

\begin{definition}
  A function $E:\zo^n\times\zo^d \to \zo^r$ is a 
  \emph{$(k,\epsilon)$-extractor} if, for all $k$-sources~$X$ on 
  $\zo^n$, we have $\sdist\big(E(X,U_{d}),U_{r}\big) \leq \epsilon$.
\end{definition}
The following theorem shows that seeded extractors exist, and the 
proof uses the probabilistic method.
\begin{theorem}\label{theorem:existence}
  For all positive integers $n$ and $k\leq n$, and all $\epsilon>0$, 
  there exists a $(k,\epsilon)$-extractor
  $E:\zo^n\times\zo^d \to \zo^r$
  with $r=k-2\log\frac{1}{\epsilon}-O(1)$
  and $d=\log(n-k)+2\log\frac{1}{\epsilon}+\Omega(1)$.
\end{theorem}
Because the $k$-source~$X$ and the uniform distribution~$U_d$ are 
independent, their min-entropies add up, for which reason the overall 
entropy of the input of $E$ is $k+d$.
The smaller $\epsilon$ is, the less randomness~$r$ can be extracted.

\begin{proof}[Idea]
  Let $E$ be sampled uniformly at random from the set of all functions 
  of the specified type.
  We want to argue that $E$ fails to be an extractor with probability 
  strictly less than one.
  Let $X$ be a $k$-source.
  By Lemma~\ref{lemma:convex}, we know that $X$ can be written as a 
  convex combination of flat $k$-sources.
  Thus, if $X$ is an extractor for all flat $k$-sources, it is an 
  extractor for~$X$.
  For this, we use the union bound, and the probability of $E$ not 
  being an extractor for~$X$ is at most a factor of at most 
  $\binom{2^n}{2^k}$ larger than the probability of $E$ not being an 
  extractor for a flat $k$-source, since that is the number of all 
  flat $k$-sources on $\zo^n$.
  Formally, we have
  \begin{align*}
    \Pr_E\big[\text{$E$ is not a $(k,\epsilon)$-extractor}\big]
    &\leq \binom{2^n}{2^k} \cdot \max_{\text{flat 
        $k$-source~$F$}}\quad \Pr_E\big[\text{$E$ is not an 
      $\epsilon$-extractor for $F$}\big]
    \\
    &\leq \left( 2^{n-k} e \right)^{2^k} \cdot 
    2^{-\Omega\left(2^k2^d\epsilon^2\right)}
    < 1
  \end{align*}
  The second inequality follows from $\binom{N}{K}\leq(Ne/K)^K$ and 
  Lemma~\ref{lemma:random} using the fact that $(F,U_d)$ has 
  min-entropy $k+d$, and the last inequality follows from the choice 
  of parameters.
\end{proof}

\section{Motivation: Derandomization}

Extractors can be used to reduce the amount of randomness required by 
randomized algorithms.
Let $A(x,\rho)$ be a randomized algorithm so that, for all inputs $x$, 
we have
\begin{equation*}
  \Pr\big(\text{$A(x,U_r)$ fails}\big)\leq\delta\,,
\end{equation*}
and let $E$ be a $(k,\epsilon)$-extractor.
Then we define a derandomized algorithm by generating the randomness 
using~$E$ and taking a majority vote over the extractor seed:
\begin{equation*}
  A'(x,\rho') \doteq \maj_{s\in\zo^d} A\big(x,E(\rho',s)\big)\,.
\end{equation*}
The main advantage of this approach is that $A'(x,Y)$ does not need a 
uniform source $Y$ anymore; a $k$-source suffices.
Then we have, for all inputs $x$,
\begin{equation*}
  \Pr(\text{$A'(x,Y)$ fails}) \leq 2 (\delta + \epsilon)\,.
\end{equation*}
This allows to simulate $A$ when there is a little uniform randomness 
and access to large string of not necessarily uniform randomness like 
key strokes.

\begin{lemma}
  For all functions $f$ so that $A(x,U_r)$ computes $f(x)$ with 
  probability at least $1-\delta$, we have that $A'(x,Y)$ computes 
  $f(x)$ with probability at least $1-2(\epsilon+\delta)$ for all 
  $k$-sources $Y$ on $\zo^n$.
\end{lemma}
\begin{proof}
  We consider the statistical distance and use the triangle 
  inequality:
  \begin{align*}
    \sdist\big(A(x,E(x,U_{d})),f(x)\big)
    &\leq d\big(A(x,E(x,U_{d})),A(x,Ur)\big) &(\text{$\leq \epsilon$ by 
      extractor property})\\
    &+ d\big(A(x,U_{r}),f(x)\big) &(\text{$\leq \delta$ by failure 
      probability})\\
    &\leq \delta + \epsilon
  \end{align*}
  Then we use Markov's inequality to compute the probability that the 
  majority vote fails:
  \begin{align*}
    \Pr_{\rho'\sim Y} \big[ \text{$\geq\frac{1}{2}$ of all $s$ are 
      bad}\big] \leq 2(\epsilon + \delta)\,.
  \end{align*}
\end{proof}
The algorithm $A'$ runs in time $2^{d}$ times the running time of~$A$ 
plus the running time of~$E$.
Thus, while Theorem~\ref{theorem:existence} guarantees that a seeded 
extractor with good parameters exists, we don't have it explicitly and 
we don't know how efficiently it can be computed.

Next time, we will see how to construct seeded extractors explicitly 
by using their strong relationship with bipartite expanders.
\end{document}
