\magnification=\magstep1
\def\dd{\partial}
\def\pd#1#2{{\dd #1 \over \dd #2}}
\def\wh#1{{\widehat{#1}}}
\def\bar#1{{\overline{#1}}}
\centerline{Mathematics 376 -- Probability and Statistics II}
\centerline{Problem Set 8 Solutions}
\centerline{April 26, 2004}
\bigskip
\noindent
A)  
\item{1)} From the linear model, $\epsilon_i = Y_i - \beta_0 - \beta_1 x_i$
is normally distributed ($N(0,\sigma^2)$).  Hence the pdf for $\epsilon_i$
is 
$$f(\epsilon) = {1\over\sqrt{2\pi\sigma^2}} e^{-\epsilon^2/2\sigma^2}$$
From observations $(x_1,y_1),\ldots,(x_n,y_n)$ of 
$Y$, the likelihood of these observations (as a function
of $\beta_0$ and $\beta_1$) is given by 
$$\eqalign{
L(\beta_0,\beta_1) &= \left({1\over \sqrt{2\pi\sigma^2}}\right)^n 
\exp\left(\left(-\sum_{i=1}^n \epsilon_i^2\right)
{1\over 2\sigma^2}\right)\cr
&=  \left({1\over \sqrt{2\pi\sigma^2}}\right)^n 
\exp\left(\left(-\sum_{i=1}^n (y_i - \beta_0 - \beta_1x_i)^2\right)
{1\over 2\sigma^2}\right)\cr}$$
\item{2)}  Continuing from the last line, we have
$$\ln(L) = {-n\over 2}\ln(2\pi\sigma^2) - 
{1\over 2\sigma^2} \sum_{i=1}^n (y_i - \beta_0 - \beta_1x_i)^2$$
Hence
$$\eqalign{
\pd{\ln(L)}{\beta_0} &= {-1\over 2\sigma^2}\sum_{i=1}^n 2(y_i - \beta_0 - \beta_1x)(-1)\cr
&={1\over \sigma^2}\left((\Sigma y_i) - n\beta_0 - (\Sigma x_i)\beta_1\right)\cr
\pd{\ln(L)}{\beta_1} &= {-1\over 2\sigma^2}\sum_{i=1}^n 2(y_i - \beta_0 - \beta_1x)(-x_i)\cr
&={1\over \sigma^2}\left((\Sigma x_iy_i) - (\Sigma x_i)\beta_0 - (\Sigma x_i^2)\beta_1\right)\cr
}$$
Setting $\pd{\ln(L)}{\beta_0} = \pd{\ln(L)}{\beta_1} = 0$ and 
multiplying by $\sigma^2$, we get exactly the same normal equations
as from the least-squares approach:
$$\eqalign{
n\beta_0 + (\Sigma x_i)\beta_1 &= \Sigma y_i\cr
(\Sigma x_i)\beta_0 + (\Sigma x_i^2)\beta_1 &= \Sigma x_iy_i\cr}$$
Thus, the maximum likelihood estimators for $\beta_0,\beta_1$ are
the same as the least-squares estimators.
\item{3)}  To check that $(\wh{\beta}_0,\wh{\beta}_1)$ gives a 
maximum of the likelihood, we continue on to the second derivative 
test:
$$\eqalign{
{\dd^2 \ln(L)\over \dd\beta_0^2} &= {-n\over \sigma^2}\cr
{\dd^2 \ln(L)\over \dd\beta_1^2} &= {-(\Sigma x_i^2)\over \sigma^2}\cr
{\dd^2 \ln(L)\over \dd\beta_1\dd\beta_0} 
&= {-(\Sigma x_i)\over \sigma^2}\cr}$$
The Hessian determinant is 
$$H = 
{\dd^2 \ln(L)\over \dd\beta_0^2}{\dd^2 \ln(L)\over \dd\beta_1^2}
-\left({\dd^2 \ln(L)\over \dd\beta_1\dd\beta_0}\right)^2
= {n(\Sigma x_i^2) - (\Sigma x_i)^2 \over \sigma^4}$$
The numerator here is $nS_{xx} > 0$  (see B below).  Hence $H > 0$ at all 
points (including the critical point found in part 2).
Furthermore, ${\dd^2 \ln(L)\over \dd\beta_0^2} = {-n\over \sigma^2} < 0$,
so $(\wh{\beta}_0,\wh{\beta}_1)$ is a local (and global) maximum.
\bigskip
\item{}
{\it Comment:}  If you missed some points on this problem, be sure
to review the second derivative test for functions of two variables
again and be ready to use it correctly on the final exam(!)
\bigskip
\noindent
B) 
\item{1)}  We have 
$$\eqalign{S_{xx} &= \Sigma (x_i - \bar{x})^2\cr
&=(\Sigma x_i^2) - 2\bar{x}(\Sigma x_i) + n\bar{x}^2\cr
&=(\Sigma x_i^2) - 2n \bar{x}^2 + n\bar{x}^2\cr
&=(\Sigma x_i^2) - n \bar{x}^2\cr}$$
Similarly,
$$\eqalign{
S_{yy} &=\Sigma (x_i - \bar{x})(y_i - \bar{y})\cr
&=(\Sigma x_iy_i) - \bar{x}(\Sigma y_i) - \bar{y}(\Sigma x_i) + n\bar{x}\bar{y}\cr
&=(\Sigma x_iy_i) - 2n \bar{x}\bar{y} + n\bar{x}\bar{y}\cr
&=(\Sigma x_iy_i) - n \bar{x}\bar{y}\cr}$$
Hence
$$\eqalign{
{S_{xy}\over S_{xx}} 
&= {(\Sigma x_iy_i) - n \bar{x}\bar{y}\over (\Sigma x_i^2) - n \bar{x}^2}\cr
&= {n(\Sigma x_iy_i) - n^2 \bar{x}\bar{y}\over n(\Sigma x_i^2) - n^2 \bar{x}^2}\cr
&= {n(\Sigma x_iy_i) - (\Sigma x_i)(\Sigma y_i)\over n(\Sigma x_i^2) - (\Sigma x_i)^2}\cr
&=\wh{\beta}_1\cr}$$
\item{2)}  Then we have 
$$\eqalign{
\bar{y} - \bar{x}\wh{\beta}_1
&=\bar{y} - \bar{x}{n(\Sigma x_iy_i) - (\Sigma x_i)(\Sigma y_i)\over n(\Sigma x_i^2) - (\Sigma x_i)^2}\cr
&={\bar{y} (n(\Sigma x_i^2) - (\Sigma x_i)^2) - \bar{x}(n(\Sigma x_iy_i) - (\Sigma x_i)(\Sigma y_i))
\over n(\Sigma x_i^2) - (\Sigma x_i)^2}\cr
&={{1\over n}(\Sigma y_i) (n(\Sigma x_i^2) - (\Sigma x_i)^2) - {1\over n}(\Sigma x_i)(n(\Sigma x_iy_i) - (\Sigma x_i)(\Sigma y_i))
\over n(\Sigma x_i^2) - (\Sigma x_i)^2}\cr
&={(\Sigma y_i)(\Sigma x_i^2) - (\Sigma x_i)(\Sigma x_iy_i)\over
n(\Sigma x_i^2) - (\Sigma x_i)^2}\cr
&=\wh{\beta}_0\cr
}$$
\bigskip
\noindent
11/6.  We have 
$SSE = \Sigma (y_i - \beta_1x_i)^2$, so to minimize, 
we compute $\pd{SSE}{\beta_1}$ and set equal to zero:
$$\eqalign{
0 &=2 \Sigma (y_i - \beta_1 x_i)(-x_i)\cr
  &= 2(\Sigma x_iy_i) - 2\beta_1 (\Sigma x_i^2)\cr
\Rightarrow \beta_1 (\Sigma x_i^2)&= \Sigma x_iy_i\cr}$$
Solving for $\beta_1$, we obtain the least-squares estimator:
$$\wh{\beta}_1 = {\Sigma x_iy_i \over \Sigma x_i^2}$$
Note that this is different from $\wh{\beta}_1$ in the case
that the regression line is not constrained to pass through
(0,0).
\bigskip
\noindent
11/11.  We substitute $\wh{\beta}_0 = \bar{y} - \wh{\beta}_1\bar{x}$
into the formula for SSE:
$$\eqalign{
SSE &= \Sigma (y_i - \wh{\beta}_0 - \wh{\beta}_1x_i)^2\cr
    &= \Sigma \left((y_i - \bar{y}) - \wh{\beta}_1(x_i - \bar{x})\right)^2\cr
    &= S_{yy} - 2\wh{\beta}_1 S_{xy} + \wh{\beta}_1^2 S_{xx}\cr}$$
But, 
$$\wh{\beta}_1^2 S_{xx} = \wh{\beta}_1\left({S_{xy}\over S_{xx}}\right) S_{xx}
=\wh{\beta}_1 S_{xy}$$
So the last two terms in the above can be combined to yield:
$$SSE = S_{yy} - \wh{\beta}_1 S_{xy}$$
which is what we wanted to show.
\bigskip
\noindent
For the other problems, see the following Maple session.
\bye