\documentclass[12pt]{article} 
%\usepackage{amsbsy} % for \boldsymbol and \pmb 
\usepackage{graphicx} % To include pdf files!
\usepackage{amsmath}
\usepackage{amsbsy}
\usepackage{amsfonts}
\usepackage[colorlinks=true, pdfstartview=FitV, linkcolor=blue, citecolor=blue, urlcolor=blue]{hyperref} % For links
\usepackage{fullpage}
%\pagestyle{empty} % No page numbers


\begin{document}
%\enlargethispage*{1000 pt} 

\begin{center}   
{\Large \textbf{STA 2101 Assignment 2}}\footnote{This assignment was prepared by  \href{http://www.utstat.toronto.edu/~brunner}{Jerry Brunner},
Department of Statistics, University of Toronto. It is licensed under a 
\href{http://creativecommons.org/licenses/by-sa/3.0/deed.en_US}
     {Creative Commons Attribution - ShareAlike 3.0 Unported License}. Use any part of it as you like and share the result freely. The \LaTeX~source code is available from the course website:
\href{http://www.utstat.toronto.edu/~brunner/oldclass/2101f19} {\texttt{http://www.utstat.toronto.edu/$^\sim$brunner/oldclass/2101f19}}}
\vspace{1 mm}
\end{center}

\noindent
The questions on this assignment are not to be handed in. They are practice for Quiz Two on Friday September 27th. There is a posted formula sheet that will be provided with the quiz. The linear algebra questions are more review.

\begin{enumerate} 


    \item Let     
\begin{tabular}{ccc}
$\mathbf{A} = 
     \left( \begin{array}{c c}
                 1 & 2 \\
                 2 & 4 
    \end{array} \right) $    &
$\mathbf{B} = 
     \left( \begin{array}{c c}
                 0 & 2 \\
                 2 & 1 
    \end{array} \right) $  &
$\mathbf{C} = 
     \left( \begin{array}{c c}
                 2 & 0 \\
                 1 & 2 
    \end{array} \right) $
\end{tabular}
    \begin{enumerate}
        \item Calculate $\mathbf{AB}$ and $\mathbf{AC}$
        \item Do we have $\mathbf{AB} = \mathbf{AC}$? Answer Yes or No.
        \item Prove $\mathbf{B} = \mathbf{C}$. Show your work. 
    \end{enumerate}

    \item Let $\mathbf{X}$ be an $n$ by $p$ matrix with $n \neq p$. Why is it incorrect to say that $(\mathbf{X^\top X})^{-1}= \mathbf{X}^{-1}\mathbf{X}^{\top -1}$?
    
    \item  \label{ss} Let $\mathbf{a}$ be an $n \times 1$ matrix of real constants. How do you know $\mathbf{a}^\top\mathbf{a}\geq 0$?

    \item The $p \times p$ matrix $\boldsymbol{\Sigma}$ is said to be \emph{positive definite} if
           $\mathbf{a}^\top \boldsymbol{\Sigma} \mathbf{a} > 0$ for all $p \times 1$ vectors $\mathbf{a} \neq \mathbf{0}$. Show that the eigenvalues of a positive definite matrix are all strictly positive. A good approach is to start with the definition of an eigenvalue and the corresponding eigenvalue:  $\boldsymbol{\Sigma}\mathbf{v} = \lambda \mathbf{v}$. Eigenvectors are typically scaled to have length one, so you may assume $\mathbf{v}^\top \mathbf{v} = 1$.

    \item  Recall the \emph{spectral decomposition} of a symmetric matrix (for example, a variance-covariance matrix).  Any such matrix $\boldsymbol{\Sigma}$ can be written as 
            $\boldsymbol{\Sigma} = \mathbf{P} \boldsymbol{\Lambda} \mathbf{P}^\top$,
where $\mathbf{P}$ is a matrix whose columns are the (orthonormal) eigenvectors of $\boldsymbol{\Sigma}$, $\boldsymbol{\Lambda}$ is a diagonal matrix of the corresponding eigenvalues, and $\mathbf{P}^\top\mathbf{P} =~\mathbf{P}\mathbf{P}^\top =~\mathbf{I}$. If $\boldsymbol{\Sigma}$ is real, the eigenvalues are real as well.

        \begin{enumerate}
           \item Let $\boldsymbol{\Sigma}$ be a square symmetric matrix with eigenvalues that are all strictly positive. 
                \begin{enumerate}
                    \item What is $\boldsymbol{\Lambda}^{-1}$?
                    \item Show 
            $\boldsymbol{\Sigma}^{-1} = \mathbf{P} \boldsymbol{\Lambda}^{-1} \mathbf{P}^\top$
                \end{enumerate}
           \item Let $\boldsymbol{\Sigma}$ be a square symmetric matrix, and this time the eigenvalues are non-negative. 
                \begin{enumerate}
                    \item What do you think $\boldsymbol{\Lambda}^{1/2}$ might be?
                    \item Define $\boldsymbol{\Sigma}^{1/2}$ as 
                    $\mathbf{P} \boldsymbol{\Lambda}^{1/2} \mathbf{P}^\top$.
                    Show $\boldsymbol{\Sigma}^{1/2}$ is symmetric.
                    \item Show $\boldsymbol{\Sigma}^{1/2}\boldsymbol{\Sigma}^{1/2} = \boldsymbol{\Sigma}$, justifying the notation.
                \end{enumerate}

           \item Now return to the situation where the eigenvalues of the square symmetric matrix $\boldsymbol{\Sigma}$ are all strictly positive. Define $\boldsymbol{\Sigma}^{-1/2}$ as $\mathbf{P} \boldsymbol{\Lambda}^{-1/2} \mathbf{P}^\top$, where the elements of the diagonal matrix $\boldsymbol{\Lambda}^{-1/2}$ are the reciprocals of the corresponding elements of $\boldsymbol{\Lambda}^{1/2}$. 
                \begin{enumerate}
                    \item Show that the inverse of $\boldsymbol{\Sigma}^{1/2}$ is $\boldsymbol{\Sigma}^{-1/2}$, justifying the notation.
                    \item Show $\boldsymbol{\Sigma}^{-1/2} \boldsymbol{\Sigma}^{-1/2} = \boldsymbol{\Sigma}^{-1}$.
                \end{enumerate}

           \item Let $\boldsymbol{\Sigma}$ be a symmetric, positive definite matrix. How do you know that $\boldsymbol{\Sigma}^{-1}$ exists?
        \end{enumerate}

% \pagebreak

    \item Let $\mathbf{X}$ be an $n \times p$ matrix of constants. The idea is that $\mathbf{X}$ is the ``design matrix" in the linear model 
$\mathbf{y} = \mathbf{X}\boldsymbol{\beta} + \boldsymbol{\epsilon}$, so this problem is really about linear regression.

        \begin{enumerate}

            \item Recall that $\mathbf{A}$ symmetric means $\mathbf{A=A^\top}$. Let $\mathbf{X}$ be an $n$ by $p$ matrix. Show that $\mathbf{X^\top X}$ is symmetric.
    
            \item Recall the definition of linear independence. The columns of 
$\mathbf{A}$ are said to be \emph{linearly dependent} if there exists a column vector $\mathbf{v} \neq \mathbf{0}$ with $\mathbf{Av} = \mathbf{0}$. If $\mathbf{Av} = \mathbf{0}$ implies $\mathbf{v} = \mathbf{0}$, the columns of $\mathbf{A}$ are said to be linearly \emph{independent}. Show that if the columns of $\mathbf{X}$ are linearly independent, then $\mathbf{X}^\top\mathbf{X}$ is positive definite.

            \item Show that if $\mathbf{X}^\top\mathbf{X}$ is positive definite then $(\mathbf{X}^\top\mathbf{X})^{-1}$ exists.
            
            \item Show that if $(\mathbf{X}^\top\mathbf{X})^{-1}$ exists then  the columns of  $\mathbf{X}$ are linearly independent.
            
        \end{enumerate}
This is a good problem because it establishes that the least squares estimator $\widehat{\boldsymbol{\beta}} = (\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top\mathbf{y}$ exists if and only if the columns of $\mathbf{X}$ are linearly independent.
 
%%%%%%%%%%%%%%%%%%%%%%%%% End of linear algebra questions %%%%%%%%%%%%%%%%%%%%%%%%% 

\vspace{3mm}
\hrule
\vspace{2mm}

\item Women and men are coming into a store according to independent Poisson processes with rates $\lambda_1$ for women and $\lambda_2$ for men. You don't have to know anything about Poisson processes to do this question. We have that the number of women and the number of men entering the store in a given time period are independent Poisson random variables, with expected values $\lambda_1$ for women and $\lambda_2$ for men. Because the Poisson process is an independent increments process, we can treat the numbers from $n$ time periods as a random sample.

Management wants to know the expected number of male customers and the expected number of female customers. Unfortunately, the total numbers of customers were recorded, but not their sex. Let $y_1, \ldots, y_n$ denote the total numbers of customers who enter the store in $n$ time periods. That's all the data we have.
    \begin{enumerate}
        \item What is the distribution of $y_i$? If you know the answer, just write it down without proof.
        \item What is the parameter space?
        \item Find the MLE of the parameter vector $(\lambda_1,\lambda_2)$. Show your work.
        \item How is this question related to the Zipper example?
    \end{enumerate}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% Large sample %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\vspace{3mm}
\hrule
\vspace{3mm}

    \item Suppose $\sqrt{n}(T_n-\theta) \stackrel{d}{\rightarrow} T$. Show $T_n \stackrel{p}{\rightarrow} \theta$. Please use Slutsky lemmas rather than definitions. Hint: Think of the sequence of constants $\frac{1}{\sqrt{n}}$ as a sequence of degenerate random variables (variance zero) that converge almost surely and hence in probability to zero. Now you can use a Slutsky lemma.

   \item Let $X_1 , \ldots, X_n$ be a random sample from a Binomial distribution with parameters $3$ and $\theta$. That is, 
\begin{displaymath}
    P(X_i = x_i) = \binom{3}{x_i} \theta^{x_i} (1-\theta)^{3-x_i},
\end{displaymath}
for $x_i=0,1,2,3$. Find the maximum likelihood estimator of $\theta$, and show that it is strongly consistent.

   \item  Let $X_1 , \ldots, X_n$ be a random sample from a continuous distribution with density
\begin{displaymath}
    f(x;\tau) = \frac{\tau^{1/2}}{\sqrt{2\pi}} \, e^{-\frac{\tau x^2}{2}},
\end{displaymath}
where the parameter $\tau>0$.  Let
\begin{displaymath}
    \widehat{\tau} = \frac{n}{\sum_{i=1}^n X_i^2}.
\end{displaymath}
Is $ \widehat{\tau}$ a consistent estimator of $\tau$? Answer Yes or No and prove your answer. Hint: You can just write down $E(X^2)$ by inspection. This is a very familiar distribution.

    \item Let $X_1, \ldots, X_n$ be a random sample from a distribution with mean $\mu$. Show that $T_n = \frac{1}{n+400}\sum_{i=1}^n X_i$ is a strongly consistent estimator of $\mu$.
    
    \item Let $X_1, \ldots, X_n$ be a random sample from a distribution with mean $\mu$ and variance $\sigma^2$. Prove that the sample variance $S^2=\frac{\sum_{i=1}^n(X_i-\overline{X})^2}{n-1}$ is a strongly consistent estimator of $\sigma^2$. 

   \item \label{randiv} Independently for $i = 1 , \ldots, n$, let 
\begin{displaymath}
    Y_i = \beta X _i + \epsilon_i,
\end{displaymath}
where $E(X_i)=E(\epsilon_i)=0$, $Var(X_i)=\sigma^2_X$,  $Var(\epsilon_i)=\sigma^2_\epsilon$, and $\epsilon_i$ is independent of $X_i$.  Let
\begin{displaymath}
    \widehat{\beta}_n = \frac{\sum_{i=1}^n X_i Y_i}{\sum_{i=1}^n X_i^2}.
\end{displaymath}
Is $ \widehat{\beta}_n$ a consistent estimator of $\beta$? Answer Yes or No and prove your answer.

   \item  In this problem, you'll use (without proof) the \emph{variance rule}, which says that if $\theta$ is a real constant and $T_1, T_2, \ldots$ is a sequence of random variables with
\begin{displaymath}
    \lim_{n \rightarrow \infty} E(T_n) = \theta \mbox{ and }
    \lim_{n \rightarrow \infty} Var(T_n) = 0,
\end{displaymath}
then $ T_n \stackrel{P}{\rightarrow} \theta$.
 
In Problem~\ref{randiv}, the independent variables are random. Here they are fixed constants, which is more standard (though a little strange if you think about it). Accordingly, let 
\begin{displaymath}
    Y_i = \beta x_i + \epsilon_i
\end{displaymath}
for $i=1, \ldots, n$, where  $\epsilon_1, \ldots, \epsilon_n$ are a random sample from a distribution with expected value zero and variance $\sigma^2$, and $\beta$ and $\sigma^2$ are unknown constants.
        \begin{enumerate}
            \item What is $E(Y_i)$?
            \item What is $Var(Y_i)$?
%            \item Find the Least Squares estimate of $\beta$ by minimizing
%                  $Q=\sum_{i=1}^n(Y_i-\beta x_i)^2$ over all values of $\beta$. Let $\widehat{\beta}_n$ denote the point at which $Q$ is minimal.
            \item Use the same estimator as in Problem~\ref{randiv}. Is $\widehat{\beta}_n$ unbiased? Answer Yes or No and show your work.
            \item Suppose that the sequence of constants $\sum_{i=1}^nx_i^2 \rightarrow \infty$ as $n \rightarrow \infty$. Does this guarantee $\widehat{\beta}_n$ will be consistent? Answer Yes or No. Show your work. % There are other conditions on the $x_i$ values that work, but this might be the best.
            \item Let 
            $\widehat{\beta}_{2,n} = \frac{\overline{Y}_n}{\overline{x}_n}$. Is $\widehat{\beta}_{2,n}$ unbiased? Consistent? Answer Yes or No to each question and show your work. Do you need a condition on the $x_i$ values ?
            \item Prove that $\widehat{\beta}_n$ is a more accurate estimator than $\widehat{\beta}_{2,n}$ in the sense that it has smaller variance. Hint: The sample variance of the explanatory variable values cannot be negative. 
        \end{enumerate}

    \item Let $X$ be a random variable with expected value $\mu$ and variance $\sigma^2$. Show $\frac{X}{n} \stackrel{p}{\rightarrow} 0$. 

   \item  Let $X_1 , \ldots, X_n$ be a random sample from a Gamma distribution with $\alpha=\beta=\theta>0$. That is, the density is
\begin{displaymath}
    f(x;\theta) = \frac{1}{\theta^\theta \Gamma(\theta)} e^{-x/\theta} x^{\theta-1}, 
\end{displaymath}
for $x>0$.  Let $\widehat{\theta} = \overline{X}_n$. Is $ \widehat{\theta}$ a consistent estimator of $\theta$? Answer Yes or No and prove your answer.

\item \label{integrateR} Here is an integral you cannot do in closed form, and numerical integration is challenging. For example, R's \texttt{integrate} function fails.

\begin{displaymath}
    \int_0^{1/2} e^{\cos(1/x)} \, dx
\end{displaymath}
Using R, approximate the integral with Monte Carlo integration, and give a 99\% confidence interval for your answer. You need to produce 3 numbers: the estimate, a lower confidence limit and an upper confidence limit. See lecture slides. \textbf{Bring your printout to the quiz.}





\end{enumerate}


\end{document}