(33) $\displaystyle\leq$
$\displaystyle\tau\left|\frac{1}{n}\sum_{i=1}^{n}\left[\mathscr{E}_{i}(\widehat{{\cal
P}}_{\perp}^{(1)}\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})\theta_{i\pi(k)}\right]\right|$
$\displaystyle\leq$
$\displaystyle\tau\left|\frac{1}{n}\sum_{i=1}^{n}\left[\mathscr{E}_{i}(\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})\theta_{i\pi(k)}\right]\right|+\tau\left|\frac{1}{n}\sum_{i=1}^{n}\left[\mathscr{E}_{i}(\widehat{{\cal
P}}^{(1)}\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})\theta_{i\pi(k)}\right]\right|,$

where the second inequality follows from triangular inequality. As before,

$\left|\frac{1}{n}\sum_{i=1}^{n}\left[\mathscr{E}_{i}(\widehat{{\cal
P}}^{(1)}\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})\theta_{i\pi(k)}\right]\right|\leq\|\widehat{\Sigma}_{\theta,\mathscr{E}}\|\|\widehat{{\cal
P}}^{(1)}\mathbf{u}_{m}^{(1)}\|\leq C\sigma_{0}\delta_{k}\sqrt{d\over n}.$

To bound the first term on the rightmost hand side of (33), write

$\widehat{\mathbf{u}}_{k}^{(q)}=\alpha_{q}\mathbf{u}_{\pi(k)}+\mathbf{v}^{(q)}$

where
$\alpha_{q}=\langle\mathbf{u}_{\pi(k)},\widehat{\mathbf{u}}_{k}\notin\\{\pi(k),m\\}\notin\\{\pi(k),m\\}\notin\\{\pi(k),m\\}^{(q)}\rangle$
and $\|\mathbf{v}^{(q)}\|=\sqrt{1-\alpha_{q}^{2}}\leq\delta_{k}$. Then

$\displaystyle\mathscr{E}_{i}(\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})$
$\displaystyle=$
$\displaystyle\alpha_{p}\mathscr{E}_{i}(\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p-1)},\mathbf{u}_{\pi(k)}^{(p)})+\mathscr{E}_{i}(\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p-1)},\mathbf{v}^{(p)})$
$\displaystyle=$
$\displaystyle\alpha_{p}\alpha_{p-1}\mathscr{E}_{i}(\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p-2)},\mathbf{u}_{\pi(k)}^{(p-1)},\mathbf{u}_{\pi(k)}^{(p)})$
$\displaystyle+\alpha_{p}\mathscr{E}_{i}(\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p-2)},\mathbf{v}^{(p-1)},\mathbf{u}_{\pi(k)}^{(p)})$
$\displaystyle+\mathscr{E}_{i}(\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p-1)},\mathbf{v}^{(p)})$
$\displaystyle=$ $\displaystyle\cdots\cdots$ $\displaystyle=$
$\displaystyle\left(\prod_{q=2}^{p}\alpha_{q}\right)\mathscr{E}_{i}(\mathbf{u}_{m}^{(1)},\mathbf{u}_{\pi(k)}^{(2)},\dots,\mathbf{u}_{\pi(k)}^{(p)})$
$\displaystyle+\cdots+\mathscr{E}_{i}(\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p-1)},\mathbf{v}^{(p)}).$

Therefore,

$\displaystyle\left|\frac{1}{n}\sum_{i=1}^{n}\left[\mathscr{E}_{i}(\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})\theta_{i\pi(k)}\right]\right|$
$\displaystyle\leq$ $\displaystyle\left|{1\over
n}\sum_{i=1}^{n}\left[\mathscr{E}_{i}(\mathbf{u}_{m}^{(1)},\mathbf{u}_{\pi(k)}^{(2)},\dots,\mathbf{u}_{\pi(k)}^{(p)})\theta_{i\pi(k)}\right]\right|+\cdots+\left|\frac{1}{n}\sum_{i=1}^{n}\left[\mathscr{E}_{i}(\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p-1)},\mathbf{v}^{(p)})\theta_{i\pi(k)}\right]\right|$
$\displaystyle\leq$ $\displaystyle\left|{1\over
n}\sum_{i=1}^{n}\left[\mathscr{E}_{i}(\mathbf{u}_{m}^{(1)},\mathbf{u}_{\pi(k)}^{(2)},\dots,\mathbf{u}_{\pi(k)}^{(p)})\theta_{i\pi(k)}\right]\right|+C_{p}\delta_{k}\|\widehat{\Sigma}_{\theta,\mathscr{E}}\|$
$\displaystyle\leq$ $\displaystyle C\left(\sigma_{0}\sqrt{\log r\over
n}+\sigma_{0}\delta_{k}\sqrt{d\over n}\right),$

so that the six term on the rightmost hand side of (26) can be upper bounded
by

$\left|\frac{1}{n}\sum_{i=1}^{n}\left[\sum_{l=1}^{r}\mathscr{E}_{i}(\widehat{{\cal
P}}_{\perp}^{(1)}\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})\sigma_{l}\theta_{il}\left(\prod_{q=1}^{p}\langle\mathbf{u}_{l}^{(q)},\widehat{\mathbf{u}}_{k}^{(q)}\rangle\right)\right]\right|\leq
C\left(\sigma_{0}\tau\sqrt{\log r\over n}+\sigma_{0}\tau\delta_{k}\sqrt{d\over
n}\right).$

Finally consider the seventh term:

$\displaystyle\left|\frac{1}{n}\sum_{i=1}^{n}\mathscr{E}_{i}(\widehat{{\cal
P}}_{\perp}^{(1)}\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})\mathscr{E}_{i}(\widehat{\mathbf{u}}_{k}^{(1)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})-\langle\widehat{{\cal
P}}_{\perp}^{(1)}\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(1)}\rangle\right|$
$\displaystyle\leq$
$\displaystyle\left|\frac{1}{n}\sum_{i=1}^{n}\mathscr{E}_{i}(\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})\mathscr{E}_{i}(\widehat{\mathbf{u}}_{k}^{(1)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})-\langle\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(1)}\rangle\right|$
$\displaystyle+\left|\frac{1}{n}\sum_{i=1}^{n}\mathscr{E}_{i}(\widehat{{\cal
P}}^{(1)}\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})\mathscr{E}_{i}(\widehat{\mathbf{u}}_{k}^{(1)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})-\langle\widehat{{\cal
P}}^{(1)}\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(1)}\rangle\right|.$

Similar to before, the second term can be bounded by

$\displaystyle\left|\frac{1}{n}\sum_{i=1}^{n}\mathscr{E}_{i}(\widehat{{\cal
P}}^{(1)}\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})\mathscr{E}_{i}(\widehat{\mathbf{u}}_{k}^{(1)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})-\sigma_{0}^{2}\langle\widehat{{\cal
P}}^{(1)}\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(1)}\rangle\right|$
$\displaystyle\leq$ $\displaystyle\|\widehat{{\cal
P}}^{(1)}\mathbf{u}_{m}^{(1)}\|\|\widehat{\Sigma}_{\mathscr{E}}-\Sigma_{\mathscr{E}}\|\leq
C\sigma_{0}^{2}\delta_{k}\sqrt{d\over n};$

the first term by

$\displaystyle\left|\frac{1}{n}\sum_{i=1}^{n}\mathscr{E}_{i}(\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})\mathscr{E}_{i}(\widehat{\mathbf{u}}_{k}^{(1)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})-\sigma_{0}^{2}\langle\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(1)}\rangle\right|$
$\displaystyle\leq$
$\displaystyle\left|\frac{1}{n}\sum_{i=1}^{n}\mathscr{E}_{i}(\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})\mathscr{E}_{i}(\mathbf{v}^{(1)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})-\sigma_{0}^{2}\langle\mathbf{u}_{m}^{(1)},\mathbf{v}^{(1)}\rangle\right|$
$\displaystyle+\left|\frac{1}{n}\sum_{i=1}^{n}\mathscr{E}_{i}(\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})\mathscr{E}_{i}(\mathbf{u}_{\pi(k)}^{(1)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})\right|$
$\displaystyle\leq$ $\displaystyle
C\delta_{k}\|\widehat{\Sigma}_{\mathscr{E}}-\Sigma_{\mathscr{E}}\|+\left|\frac{1}{n}\sum_{i=1}^{n}\mathscr{E}_{i}(\mathbf{u}_{m}^{(1)},\mathbf{u}_{\pi(k)}^{(2)},\cdots,\mathbf{u}_{\pi(k)}^{(p)})\mathscr{E}_{i}(\mathbf{u}_{\pi(k)}^{(1)},\cdots,\mathbf{u}_{\pi(k)}^{(p)})\right|$
$\displaystyle+\cdots+\left|\frac{1}{n}\sum_{i=1}^{n}\mathscr{E}_{i}(\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p)})\mathscr{E}_{i}(\mathbf{u}_{\pi(k)}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\dots,\widehat{\mathbf{u}}_{k}^{(p-1)},\mathbf{v}^{(p)})\right|$
$\displaystyle\leq$ $\displaystyle C\left(\sigma_{0}^{2}\delta_{k}\sqrt{d\over
n}+\sigma_{0}^{2}\sqrt{\log r\over n}\right).$

Putting all seven upper bounds together, we have

$\max_{l\notin\pi([k])}\tilde{\Sigma}(\widehat{{\cal
P}}_{\perp}^{(1)}\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)},\widehat{\mathbf{u}}_{k}^{(1)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)})\leq
C\left(\tau^{2}\delta_{k}^{2}+(\tau^{2}\delta_{k}^{p-1}+\sigma_{0}\tau+\sigma_{0}^{2})\sqrt{\log
r\over n}\right).$

###  B.4 Finishing Up

We first verify (18). Note that

$\displaystyle(\Sigma-\Sigma_{\mathscr{E}})(\widehat{\mathbf{u}}_{k}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)},\widehat{\mathbf{u}}_{k}^{(1)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)})$
$\displaystyle=$
$\displaystyle\sum_{l=1}^{r}\left(\sigma_{l}^{2}\prod_{q=1}^{p}\langle\mathbf{u}_{l}^{(q)},\widehat{\mathbf{u}}_{k}^{(q)}\rangle^{2}\right)$
$\displaystyle\leq$ $\displaystyle\max_{1\leq l\leq
r}\left\\{\sigma_{l}^{2}\left|\prod_{q=1}^{p}\langle\mathbf{u}_{l}^{(q)},\widehat{\mathbf{u}}_{k}^{(q)}\rangle\right|\right\\}\cdot\left(\sum_{l=1}^{r}\left|\prod_{q=1}^{p}\langle\mathbf{u}_{l}^{(q)},\widehat{\mathbf{u}}_{k}^{(q)}\rangle\right|\right)$
$\displaystyle\leq$ $\displaystyle\max_{1\leq l\leq
r}\left\\{\sigma_{l}^{2}\left|\prod_{q=1}^{p}\langle\mathbf{u}_{l}^{(q)},\widehat{\mathbf{u}}_{k}^{(q)}\rangle\right|\right\\}.$

where the last inequality follows from Cauchy-Schwartz inequality. Therefore,
by definition,

$\sigma_{\pi(k)}^{2}\geq(\Sigma-\Sigma_{\mathscr{E}})(\widehat{\mathbf{u}}_{k}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)},\widehat{\mathbf{u}}_{k}^{(1)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)})$
(34)

On the other hand,

$\displaystyle(\Sigma-\Sigma_{\mathscr{E}})(\widehat{\mathbf{u}}_{k}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)},\widehat{\mathbf{u}}_{k}^{(1)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)})$
$\displaystyle\geq$
$\displaystyle\tilde{\Sigma}(\widehat{\mathbf{u}}_{k}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)},\widehat{\mathbf{u}}_{k}^{(1)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)})$
$\displaystyle-\sup_{\|\mathbf{w}^{(q)}\|\leq 1,1\leq q\leq
p}\left|(\widehat{\Sigma}-\Sigma)(\widehat{{\cal
P}}_{\perp}^{(1)}\mathbf{w}^{(1)},\dots,\widehat{{\cal
P}}_{\perp}^{(p)}\mathbf{w}^{(p)},\widehat{{\cal
P}}_{\perp}^{(1)}\mathbf{w}^{(1)},\dots,\widehat{{\cal
P}}_{\perp}^{(p)}\mathbf{w}^{(p)})\right|$ $\displaystyle\geq$
$\displaystyle\tilde{\Sigma}(\widehat{{\cal
P}}_{\perp}^{(1)}\mathbf{u}_{l_{\ast}}^{(1)},\ldots,\widehat{{\cal
P}}_{\perp}^{(p)}\mathbf{u}_{l_{\ast}}^{(p)},\widehat{{\cal
P}}_{\perp}^{(1)}\mathbf{u}_{l_{\ast}}^{(1)},\ldots,\widehat{{\cal
P}}_{\perp}^{(p)}\mathbf{u}_{l_{\ast}}^{(p)})$
$\displaystyle-\sup_{\|\mathbf{w}^{(q)}\|\leq 1,1\leq q\leq
p}\left|(\widehat{\Sigma}-\Sigma)(\widehat{{\cal
P}}_{\perp}^{(1)}\mathbf{w}^{(1)},\dots,\widehat{{\cal
P}}_{\perp}^{(p)}\mathbf{w}^{(p)},\widehat{{\cal
P}}_{\perp}^{(1)}\mathbf{w}^{(1)},\dots,\widehat{{\cal
P}}_{\perp}^{(p)}\mathbf{w}^{(p)})\right|$ $\displaystyle\geq$
$\displaystyle(\Sigma-\Sigma_{\mathscr{E}})(\widehat{{\cal
P}}_{\perp}^{(1)}\mathbf{u}_{l_{\ast}}^{(1)},\ldots,\widehat{{\cal
P}}_{\perp}^{(p)}\mathbf{u}_{l_{\ast}}^{(p)},\widehat{{\cal
P}}_{\perp}^{(1)}\mathbf{u}_{l_{\ast}}^{(1)},\ldots,\widehat{{\cal
P}}_{\perp}^{(p)}\mathbf{u}_{l_{\ast}}^{(p)})$
$\displaystyle-2\sup_{\|\mathbf{w}^{(q)}\|\leq 1,1\leq q\leq
p}\left|(\widehat{\Sigma}-\Sigma)(\widehat{{\cal
P}}_{\perp}^{(1)}\mathbf{w}^{(1)},\dots,\widehat{{\cal
P}}_{\perp}^{(p)}\mathbf{w}^{(p)},\widehat{{\cal
P}}_{\perp}^{(1)}\mathbf{w}^{(1)},\dots,\widehat{{\cal
P}}_{\perp}^{(p)}\mathbf{w}^{(p)})\right|$

where $l_{\ast}$ is the index such that $\sigma_{l_{\ast}}^{2}=\tau^{2}$.
Following the same derivation as before, we have

$\displaystyle(\Sigma-\Sigma_{\mathscr{E}})(\widehat{\mathbf{u}}_{k}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)},\widehat{\mathbf{u}}_{k}^{(1)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)})\geq\tau^{2}\left(1-C\sqrt{r\over
n}-C\delta_{k}\right).$

Together with (34), we get

$\sigma_{\pi(k)}^{2}\geq\tau^{2}\left(1-C\sqrt{r\over n}-C\delta_{k}\right).$

Combing the lower bound for
$\tilde{\Sigma}(\widehat{\mathbf{u}}_{k}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)},\widehat{\mathbf{u}}_{k}^{(1)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)})$
and upper bound for $\tilde{\Sigma}(\widehat{{\cal
P}}_{\perp}^{(1)}(\widehat{\mathbf{u}}_{k}^{(1)}-\langle\widehat{\mathbf{u}}_{k}^{(1)},\mathbf{u}_{\pi(k)}^{(1)}\rangle\mathbf{u}_{\pi(k)}^{(1)}),\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)},\widehat{\mathbf{u}}_{k}^{(1)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)})$,
we have

$\displaystyle\eta_{k}$ $\displaystyle\leq$
$\displaystyle{C\tau^{2}\delta_{k}^{2}+C(\sigma_{0}^{2}+\sigma_{0}\tau)\sqrt{d\over
n}\over\tau^{2}\left(1-C\delta_{k}^{4}-C\sqrt{r\over
n}\right)-C(\sigma_{0}\tau+\sigma_{0}^{2})\sqrt{d\over
n}-C\sigma_{0}^{2}\delta_{k}^{2}}$ $\displaystyle\leq$ $\displaystyle
C\delta_{k}^{2}+C\left({\sigma_{0}\over\tau}+{\sigma_{0}^{2}\over\tau^{2}}\right)\sqrt{d\over
n}$ $\displaystyle\leq$ $\displaystyle
C\left({\sigma_{0}\over\sigma_{\pi(k)}}+{\sigma_{0}^{2}\over\sigma_{\pi(k)}^{2}}\right)\sqrt{d\over
n}=\delta_{k}.$

Similarly, Combing the lower bound for
$\tilde{\Sigma}(\widehat{\mathbf{u}}_{k}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)},\widehat{\mathbf{u}}_{k}^{(1)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)})$
and upper bound for $\max_{l\notin\pi([k])}\tilde{\Sigma}(\widehat{{\cal
P}}_{\perp}^{(1)}\mathbf{u}_{m}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)},\widehat{\mathbf{u}}_{k}^{(1)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)})$,
we get

$\displaystyle\tilde{\eta}_{k}$ $\displaystyle\leq$
$\displaystyle{C\left(\tau^{2}\delta_{k}^{2}+(\tau^{2}\delta_{k}^{p-1}+\sigma_{0}\tau+\sigma_{0}^{2})\sqrt{\log
r\over n}\right)\over\tau^{2}\left(1-C\delta_{k}^{4}-C\sqrt{r\over
n}\right)-C(\sigma_{0}\tau+\sigma_{0}^{2})\sqrt{d\over
n}-C\sigma_{0}^{2}\delta_{k}^{2}}$ $\displaystyle\leq$ $\displaystyle
C\delta_{k}^{2}+C\left(\delta_{k}+{\sigma_{0}\over\tau}+{\sigma_{0}^{2}\over\tau^{2}}\right)\sqrt{\log
r\over n}$ $\displaystyle\leq$ $\displaystyle
C\delta_{k}^{2}+C\left({\sigma_{0}\over\sigma_{\pi(k)}}+{\sigma_{0}^{2}\over\sigma_{\pi(k)}^{2}}\right)\sqrt{\log
r\over n}=\tilde{\delta}_{k}.$

##  Appendix C Proof of Theorems 4.1 and 4.2

Note that Theorem 4.1 can be viewed as special case of Theorem 4.2 and it
suffices to prove Theorem 4.2. As before, we only need to consider the case
when $q=1$. Write

$\mathbf{w}=\langle\mathbf{w},\mathbf{u}_{\pi(k)}^{(1)}\rangle\mathbf{u}_{\pi(k)}^{(1)}+\tilde{\mathbf{w}}.$

Then

$\langle\tilde{\mathbf{u}}_{k}^{(1)},\mathbf{w}\rangle-\langle\mathbf{u}_{\pi(k)}^{(1)},\mathbf{w}\rangle=\langle\mathbf{w},\mathbf{u}_{\pi(k)}^{(1)}\rangle\left(\langle\tilde{\mathbf{u}}_{k}^{(1)},\mathbf{u}_{\pi(k)}^{(1)}\rangle-1\right)+\langle\tilde{\mathbf{u}}_{k}^{(1)},\tilde{\mathbf{w}}\rangle.$

Under the assumption $d=o(n^{1/2})$, by Lemma 1 and Theorem 3.2, it is not
hard to see that

$\displaystyle\sin\angle(\tilde{\mathbf{u}}_{k}^{(1)},\mathbf{u}_{\pi(k)}^{(1)})=O_{p}\left(\sqrt{\frac{d}{n}}\right),$
(35)

so

$1-\langle\tilde{\mathbf{u}}_{k}^{(1)},\mathbf{u}_{\pi(k)}^{(1)}\rangle\leq
1-\langle\tilde{\mathbf{u}}_{k}^{(1)},\mathbf{u}_{\pi(k)}^{(1)}\rangle^{2}=\sin^{2}\angle(\tilde{\mathbf{u}}_{k}^{(1)},\mathbf{u}_{\pi(k)}^{(1)})=O_{p}\left(\frac{d}{n}\right)=o_{p}(n^{-1/2}).$

Therefore it suffices to prove that

$\sqrt{n}\left(\langle\tilde{\mathbf{u}}_{k}^{(1)},\tilde{\mathbf{w}}\rangle-\langle\mathbf{u}_{\pi(k)}^{(1)},\tilde{\mathbf{w}}\rangle\right)\to_{d}N\left(0,\|\tilde{\mathbf{w}}\|^{2}\left({\sigma_{0}^{2}\over\sigma_{\pi(k)}^{2}}+{\sigma_{0}^{4}\over\sigma_{\pi(k)}^{4}}\right)\right).$

Recall that $\tilde{\mathbf{u}}_{k}^{(1)}$ is the leading eigenvector of

$\widehat{\Sigma}(\cdot,\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)},\cdot,\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)}),$

which is the same as the leading eigenvector of

$\tilde{\Sigma}(\cdot,\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)},\cdot,\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)}),$

so

$\displaystyle\tilde{\mathbf{u}}_{k}^{(1)}=\frac{\tilde{\Sigma}(\cdot,\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)},\tilde{\mathbf{u}}_{k}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)})}{\tilde{\Sigma}(\tilde{\mathbf{u}}_{k}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)},\tilde{\mathbf{u}}_{k}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)})},$

which implies

$\displaystyle\langle\tilde{\mathbf{u}}_{k}^{(1)},\tilde{\mathbf{w}}\rangle=\frac{\tilde{\Sigma}(\tilde{\mathbf{w}},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)},\tilde{\mathbf{u}}_{k}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)})}{\tilde{\Sigma}(\tilde{\mathbf{u}}_{k}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)},\tilde{\mathbf{u}}_{k}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)})}.$

We start with the nominator. Following an identical argument as that for (26)
in Subsection B.3, we have

$\displaystyle\tilde{\Sigma}(\tilde{\mathbf{w}},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)},\tilde{\mathbf{u}}_{k}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)})-\tilde{\Sigma}(\tilde{\mathbf{w}},\mathbf{u}_{\pi(k)}^{(2)},\dots,\mathbf{u}_{\pi(k)}^{(p)},\mathbf{u}_{\pi(k)}^{(1)},\dots,\mathbf{u}_{\pi(k)}^{(p)})$
$\displaystyle\leq\ $ $\displaystyle
C\left(\tau^{2}\delta_{k}^{2}+\tau^{2}\delta^{p+1}\sqrt{r\over
n}+\tau^{2}\delta^{p-1}\sqrt{r\over
n}+\tau^{2}\delta_{k}^{2(p-1)}+\sigma_{0}\tau\delta_{k}\sqrt{d\over
n}\right)=o(n^{-1/2})$

with probability tending to one. So

$\displaystyle\tilde{\Sigma}(\tilde{\mathbf{w}},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)},\tilde{\mathbf{u}}_{k}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)})$
$\displaystyle=$
$\displaystyle\tilde{\Sigma}(\tilde{\mathbf{w}},\mathbf{u}_{\pi(k)}^{(2)},\dots,\mathbf{u}_{\pi(k)}^{(p)},\mathbf{u}_{\pi(k)}^{(1)},\dots,\mathbf{u}_{\pi(k)}^{(p)})+o_{p}(n^{-1/2})$
$\displaystyle=$
$\displaystyle\sigma_{\pi(k)}\frac{1}{n}\sum_{i=1}^{n}\theta_{\pi(k)i}\mathscr{E}_{i}(\tilde{\mathbf{w}},\mathbf{u}_{\pi(k)}^{(2)},\dots,\mathbf{u}_{\pi(k)}^{(p)})$
$\displaystyle+\frac{1}{n}\sum_{i=1}^{n}\mathscr{E}_{i}(\mathbf{u}_{\pi(k)}^{(1)},\dots,\mathbf{u}_{\pi(k)}^{(p)})\mathscr{E}_{i}(\tilde{\mathbf{w}},\mathbf{u}_{\pi(k)}^{(2)},\dots,\mathbf{u}_{\pi(k)}^{(p)})-\mathbf{u}_{\pi(k)}^{(1)}+o_{p}(n^{-1/2})$
$\displaystyle\to_{d}$ $\displaystyle
N(0,(\sigma_{0}^{4}+\sigma_{0}^{2}\sigma_{\pi(k)}^{2})\|\tilde{\mathbf{w}}\|^{2}).$

On the other hand, similar to the previous section,

$\tilde{\Sigma}(\tilde{\mathbf{u}}_{k}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)},\tilde{\mathbf{u}}_{k}^{(1)},\widehat{\mathbf{u}}_{k}^{(2)},\ldots,\widehat{\mathbf{u}}_{k}^{(p)})=\sigma_{\pi(k)}^{2}(1+o_{p}(1)).$

Theorem 4.2 then follows from Slutsky’s Theorem.

The claim in Theorem 4.1 that

$\displaystyle\sqrt{n}\left[{\rm vec}(\tilde{\mathbf{U}}^{(q)})-{\rm
vec}(\mathbf{U}^{(q)}_{\pi})\right]$
$\displaystyle\overset{d}{\to}N\left(0,{\rm
diag}\left(\left(\frac{\sigma_{0}^{2}}{\sigma_{\pi(1)}^{2}}+\frac{\sigma_{0}^{4}}{\sigma_{\pi(1)}^{4}}\right){\cal
P}_{\mathbf{u}_{\pi(1)}^{(q)}}^{\perp},\ldots,\left(\frac{\sigma_{0}^{2}}{\sigma_{\pi(r)}^{2}}+\frac{\sigma_{0}^{4}}{\sigma_{\pi(r)}^{4}}\right){\cal
P}_{\mathbf{u}_{\pi(r)}^{(q)}}^{\perp}\right)\right)$

follows from Theorem 4.2 and the fact that

$\sigma_{\pi(k)}\frac{1}{n}\sum_{i=1}^{n}\theta_{\pi(k)i}\mathscr{E}_{i}(\cdot,\mathbf{u}_{\pi(k)}^{(2)},\dots,\mathbf{u}_{\pi(k)}^{(p)})+\frac{1}{n}\sum_{i=1}^{n}\mathscr{E}_{i}(\mathbf{u}_{\pi(k)}^{(1)},\dots,\mathbf{u}_{\pi(k)}^{(p)})\mathscr{E}_{i}(\cdot,\mathbf{u}_{\pi(k)}^{(2)},\dots,\mathbf{u}_{\pi(k)}^{(p)})-\mathbf{u}_{\pi(k)}^{(1)}$

are independent for any $k_{1}\neq k_{2}$.

##  Appendix D Proof of Theorems 4.3 and 4.4

Theorem 4.3 is a special case of Theorem 4.4 and it suffices to prove the
latter. We first need to introduce a number of notations. Denote

$\displaystyle\bar{\mathbb{P}}(\cdot):=\mathbb{P}(\cdot|\mathscr{X}_{n/2+1},\mathscr{X}_{n/2+2},\dots,\mathscr{X}_{n}),$
(36)

and

$\displaystyle\bar{\mathbb{E}}(\cdot):=\mathbb{E}(\cdot|\mathscr{X}_{n/2+1},\mathscr{X}_{n/2+2},\dots,\mathscr{X}_{n}),$
(37)

the conditional probability and expectation given
$\\{\mathscr{X}_{n/2+1},\mathscr{X}_{n/2+2},\dots,\mathscr{X}_{n}\\},$
respectively.

Write

$\widehat{{\cal
P}}_{j}^{(q)[1]}:=\check{\mathbf{u}}_{j}^{(q)[1]}\otimes\check{\mathbf{u}}_{j}^{(q)[1]},\qquad{\cal
P}_{j}^{(q)}=\mathbf{u}_{j}^{(q)}\otimes\mathbf{u}_{j}^{(q)}={\cal
P}_{\mathbf{u}_{j}^{(q)}},$ ${\cal
C}_{j}^{(q)}:=\frac{1}{\sigma_{j}^{2}}\left(I_{d_{q}}-\mathbf{u}_{j}^{(q)}\otimes\mathbf{u}_{j}^{(q)}\right),$
$\displaystyle
z_{jk}:=\mathscr{X}_{k}\times_{1}\widehat{\mathbf{u}}_{j}^{(1)[2]}\dots\times_{q-1}\widehat{\mathbf{u}}_{j}^{(q-1)[2]}\times_{q+1}\widehat{\mathbf{u}}_{j}^{(q+1)[2]}\dots\times_{p}\widehat{\mathbf{u}}_{j}^{(p)[2]},\qquad
k=1,\dots,n/2,$

and

$\displaystyle
M_{j}^{(q)[1]}:=\sum_{l=1}^{r}\sigma_{l}^{2}\left(\prod_{q^{\prime}\neq
q}\langle\mathbf{u}_{l}^{(q^{\prime})},\widehat{\mathbf{u}}_{j}^{(q^{\prime})[2]}\rangle\right)^{2}\mathbf{u}_{l}^{(1)}\otimes\mathbf{u}_{l}^{(1)}+I_{d_{q}}=\sum_{l=1}^{r}\tilde{\sigma}_{l}^{2}\mathbf{u}_{l}^{(1)}\otimes\mathbf{u}_{l}^{(1)}+I_{d_{q}},$

where

$\displaystyle\tilde{\sigma}_{l}^{2}=\sigma_{l}^{2}\left(\prod_{q^{\prime}\neq
q}\langle\mathbf{u}_{l}^{(q^{\prime})},\widehat{\mathbf{u}}_{j}^{(q^{\prime})[2]}\rangle\right)^{2}.$

Furthermore, let

$\displaystyle{\cal C}_{j}^{(q)[1]}=\sum_{l\neq
j}\frac{1}{\tilde{\sigma}_{j}^{2}-\tilde{\sigma}_{l}^{2}}\mathbf{u}_{l}^{(q)}\otimes\mathbf{u}_{l}^{(q)}+\frac{1}{\tilde{\sigma}_{j}^{2}}\left(I_{d_{q}}-\sum_{l=1}^{r}\mathbf{u}_{l}^{(q)}\otimes\mathbf{u}_{l}^{(q)}\right),$
$\displaystyle{\cal L}_{j}^{(q)[1]}=\frac{2}{n}\sum_{k=1}^{n/2}\left({\cal
C}^{(q)[1]}z_{jk}\otimes{\cal P}_{j}^{(q)}z_{jk}+{\cal
P}_{j}^{(q)}z_{jk}\otimes{\cal C}^{(q)[1]}z_{jk}\right),$ $\displaystyle{\cal
S}_{j}^{(q)[1]}=\widehat{{\cal P}}_{j}^{(q)[1]}-{\cal P}_{j}^{(q)}-{\cal
L}_{j}^{(q)[1]}.$

We use calligraphic capital letters for ${\cal P}_{j}^{(q)}$, $\widehat{{\cal
P}}_{j}^{(q)[1]}$, ${\cal C}_{j}^{(q)}$, ${\cal C}_{j}^{(q)[1]}$, ${\cal
L}_{j}^{(q)[1]}$ and ${\cal S}_{j}^{(q)[1]}$ to remind the readers that they
are matrices with specific definitions.

Define

$\displaystyle b_{j}^{(q)[1]}:=\left\langle\bar{\mathbb{E}}({\cal
S}_{j}^{(q)[1]})\mathbf{u}_{j}^{(q)},\mathbf{u}_{j}^{(q)}\right\rangle.$ (38)

and $b_{j}^{(q)[2]}$ is similarly defined. Finally, we define

$\displaystyle
b_{j}^{(q)}=\frac{\left\|\check{\mathbf{u}}_{j}^{(q),[1]}+\check{\mathbf{u}}_{j}^{(q),[2]}\right\|}{\sqrt{1+b_{j}^{(q)[1]}}+\sqrt{1+b_{j}^{(q)[2]}}}-1..$
(39)

The proof is rather involved and we shall break it into several steps.

##### Step 1.

We shall represent linear forms of $\check{\mathbf{u}}_{j}^{(q),[1]}$ as
bilinear forms of $\widehat{{\cal P}}_{j}^{(q)[1]}$, and prove that

$\displaystyle\sqrt{n}\left((1+b_{j}^{(q)})\langle\check{\mathbf{u}}^{(q)}_{j},\mathbf{v}\rangle-\langle\mathbf{u}^{(q)}_{\pi(j)},\mathbf{v}\rangle\right)\to_{d}N\left(0,\left(\frac{\sigma_{0}^{2}}{\sigma_{\pi(j)}^{2}}+\frac{\sigma_{0}^{4}}{\sigma_{\pi(j)}^{4}}\right)\|{\cal
P}_{\mathbf{u}_{\pi(j)}^{(q)}}^{\perp}\mathbf{v}\|^{2}\right).$ (40)

##### Step 2.

We prove that

$\displaystyle\langle\check{\mathbf{u}}_{j}^{(q)},\mathbf{u}_{\pi(j)}^{(q)}\rangle=\frac{1}{\sqrt{1+\frac{d_{q}}{n}\left(\frac{\sigma_{0}^{2}}{\sigma_{\pi(j)}^{2}}+\frac{\sigma_{0}^{4}}{\sigma_{\pi(j)}^{4}}\right)}}+O_{p}\left(\frac{d^{3/2}}{n^{3/2}}\right).$
(41)

Notice that by letting $\mathbf{v}=\mathbf{u}^{(q)}_{\pi(j)}$, (40) implies

$\displaystyle(1+b_{j}^{(q)})\langle\check{\mathbf{u}}_{j}^{(q)},\mathbf{u}_{\pi(j)}^{(q)}\rangle-1=o_{p}(n^{-1/2}).$

Combine it with (41), we immediately have

$\displaystyle
b_{j}^{(q)}=\sqrt{1+\frac{d_{q}}{n}\left(\frac{\sigma_{0}^{2}}{\sigma_{\pi(j)}^{2}}+\frac{\sigma_{0}^{4}}{\sigma_{\pi(j)}^{4}}\right)}-1+O_{p}\left(\frac{d^{3/2}}{n^{3/2}}\right)+o_{p}\left(\frac{1}{\sqrt{n}}\right),$

##### Step 3.

Finally, we show that

$\displaystyle\sqrt{n}\left(\widehat{b}_{j}^{(q)}-b_{j}^{(q)}\right)\overset{p}{\to}0.$
(42)

For simplicity, in the rest of the proof we shall assume without loss of
generality that the permutation $\pi$ that matches
$\widehat{\mathbf{u}}_{j}^{(q)}$ to $\mathbf{u}_{\pi(j)}^{(1)}$ is the
identity. We shall also make repeated use of the following facts, oftentimes
without explicit mentioning.

Similar to the proof of Theorem 3.2, write

$\widehat{\Sigma}_{\mathscr{E}}^{[1]}={2\over
n}\sum_{i=1}^{n/2}\mathscr{E}_{i}\otimes\mathscr{E}_{i},\qquad\widehat{\Sigma}_{\theta,\mathscr{E}}^{[1]}={2\over
n}\sum_{i=1}^{n/2}\theta_{i}\otimes\mathscr{E}_{i}.$

By Lemmas 1 and 2, with probability tending to one,

$\displaystyle\sigma_{0}^{-2}\|\widehat{\Sigma}_{\mathscr{E}}^{[1]}-\mathscr{I}\|\leq
C\sqrt{d\over n},\
\sigma_{0}^{-1}\|\widehat{\Sigma}_{\theta,\mathscr{E}}^{[1]}\|\leq
C\sqrt{d\over n}.$ (43)

Let

$\Delta^{(q),[2]}=\widehat{\mathbf{u}}_{j}^{(q),[2]}-\mathbf{u}_{j}^{(q)}.$

By Theorem 3.2, we have

$\displaystyle\max_{q\in[p]}\|\Delta^{(q),[2]}\|=O_{p}\left(\sqrt{\frac{d}{n}}\right)$

Moreover, under the assumption $d=o(n)$, by Lemma 1 and Theorem 3.2, it is not
hard to see that

$\displaystyle\sin\angle(\check{\mathbf{u}}_{j}^{(1)[1]},\mathbf{u}_{j}^{(1)})=O_{p}\left(\sqrt{\frac{d}{n}}\right).$

Denote

$\delta^{(1)[2]}:=\max\left\\{\max_{q\in[p]}\|\Delta^{(q),[2]}\|,\|\check{\mathbf{u}}_{j}^{(1)[1]}-\mathbf{u}_{j}^{(1)}\|\right\\},$

combine the two bounds above, we have that under the assumptions for Theorem
4.4,

$\displaystyle\delta^{(1)[2]}=O_{p}\left(\sqrt{\frac{d}{n}}\right).$ (44)

###  D.1 Step 1.

Without loss of generality, for this step we assume $\sigma_{0}=1$. We only
need to prove for the case $q=1$, so within this step, we shall also write
${\cal P}_{j}={\cal P}_{j}^{(1)}={\cal P}_{\mathbf{u}_{j}^{(1)}}$ and ${\cal
C}_{j}={\cal C}_{j}^{(1)}=\frac{1}{\sigma_{j}^{2}}\left(I-{\cal P}_{j}\right)$
for simplicity.

Define

$\displaystyle\eta_{jk}(\mathbf{v}):=\langle y_{jk},{\cal
P}_{j}\mathbf{u}_{j}^{(1)}\rangle\langle y_{jk},{\cal
C}_{j}\mathbf{v}\rangle,$

where

$\displaystyle
y_{jk}:=\mathscr{X}_{k}\times_{2}\mathbf{u}_{j}^{(2)}\dots\times_{p}\mathbf{u}_{j}^{(p)}=\sigma_{j}\theta_{jk}\mathbf{u}_{j}^{(1)}+\mathscr{E}_{k}\times_{2}\mathbf{u}_{j}^{(2)}\dots\times_{p}\mathbf{u}_{j}^{(p)}.$

Recall that $\widehat{{\cal
P}}_{j}^{(1)[1]}:=\check{\mathbf{u}}_{j}^{(1),[1]}\otimes\check{\mathbf{u}}_{j}^{(1),[1]}$
and $\widehat{{\cal
P}}_{j}^{(1)[2]}:=\check{\mathbf{u}}_{j}^{(1),[2]}\otimes\check{\mathbf{u}}_{j}^{(1),[2]}$.
Define

$\displaystyle\rho_{j}(\mathbf{v})^{[1]}$
$\displaystyle:=\left\langle((\widehat{{\cal
P}}_{j}^{(1)[1]}-(1+b_{j}^{(1)[1]}){\cal
P}_{j})\mathbf{u}_{j}^{(1)},\mathbf{v}\right\rangle,\mathbf{v}\in\mathbb{H}.$
(45) $\displaystyle\rho_{j}(\mathbf{v})^{[2]}$
$\displaystyle:=\left\langle((\widehat{{\cal
P}}_{j}^{[2]}-(1+b_{j}^{(1)[2]}){\cal
P}_{j})\mathbf{u}_{j}^{(1)},\mathbf{v}\right\rangle,\mathbf{v}\in\mathbb{H}.$
(46)

Equation (6.6) in Koltchinskii and Lounici (2014) provides the representation
of linear forms of $\check{\mathbf{u}}_{j}^{(1),[1]}$ in terms of
$\rho_{j}(\mathbf{v})^{[1]}$ and $\rho_{j}(\mathbf{v})^{[2]}$:

$\displaystyle\sqrt{n}\left\langle\frac{1}{2}\left[\check{\mathbf{u}}_{j}^{(1),[1]}+\check{\mathbf{u}}_{j}^{(1),[2]}-\left(\sqrt{1+b_{j}^{(1)[1]}}+\sqrt{1+b_{j}^{(1)[2]}}\right)\mathbf{u}_{j}^{(1)}\right],\mathbf{v}\right\rangle$
$\displaystyle=$
$\displaystyle\sum_{h=1}^{2}\sqrt{n}\bigg{[}\frac{\rho_{j}(\mathbf{v})^{[h]}}{\sqrt{1+b_{j}^{(1)[h]}+\rho_{j}(\mathbf{v})^{[h]}}}$
$\displaystyle+\frac{\sqrt{1+b_{j}^{(1)[h]}}}{\sqrt{1+b_{j}^{(1)[h]}+\rho_{j}(\mathbf{v})^{[h]}}(\sqrt{1+b_{j}^{(1)[h]}+\rho_{j}(\mathbf{v})^{[h]}}+\sqrt{1+b_{j}^{(1)[h]}})}\rho_{j}(u_{j}^{(1)})^{[h]}\langle\mathbf{u}_{j}^{(1)},\mathbf{v}\rangle\bigg{]}.$
(47)

We shall make use of the following lemma:

######  Lemma 3.

Under the same assumptions in Theorem 4.4, for any
$\mathbf{u},\mathbf{v}\in\mathbb{R}^{d_{q}}$,

$\displaystyle\sqrt{\frac{n}{2}}\bigg{\langle}\left[\widehat{{\cal
P}}_{j}^{(q)[1]}-(1+b_{j}^{(q)[1]}){\cal
P}_{j}^{(q)}\right]\mathbf{u},\mathbf{v}\bigg{\rangle}$
$\displaystyle\quad-{\sqrt{2\over n}}\sum_{k=1}^{n/2}\left[\langle
y_{jk}^{(q)},{\cal P}_{j}^{(q)}\mathbf{v}\rangle\langle y_{jk}^{(q)},{\cal
C}_{j}^{(q)}\mathbf{u}\rangle+\langle y_{jk}^{(q)},{\cal
P}_{j}^{(q)}\mathbf{u}\rangle\langle y_{jk}^{(q)},{\cal
C}_{j}^{(q)}\mathbf{v}\rangle\right]=o_{p}\left(\|\mathbf{u}\|\|\mathbf{v}\|\right).$

Further more, there exists universal constant $C$ such that
$\mathbb{P}(|b_{j}^{(q)[1]}|\leq C{d}/{n})\to 1$ as $n\to\infty$.

Following from Lemma 3, observe that ${\cal C}_{j}\mathbf{u}_{j}^{(1)}=0$, we
have

$\displaystyle\sqrt{\frac{n}{2}}\rho_{j}(\mathbf{v})^{[1]}-\sqrt{\frac{2}{n}}\sum_{k=1}^{\frac{n}{2}}\eta_{jk}(\mathbf{v})\overset{p}{\to}0,$
(48)
$\displaystyle\sqrt{\frac{n}{2}}\rho_{j}(\mathbf{v})^{[2]}-\sqrt{\frac{2}{n}}\sum_{k=\frac{n}{2}+1}^{n}\eta_{jk}(\mathbf{v})\overset{p}{\to}0.$
(49)

If we can show that for any $\mathbf{v}\in\mathbb{R}^{d_{1}}$,

$\displaystyle\frac{1}{\sqrt{n}}\sum_{k=1}^{n}\eta_{jk}(\mathbf{v})\overset{d}{\to}N\left(0,\frac{1+\sigma_{j}^{2}}{\sigma_{j}^{4}}\|{\cal
P}_{j}^{\perp}\mathbf{v}\|^{2}\right),$ (50)

then combining with the facts that $b_{j}^{(1)[h]}\overset{p}{\to}0$ and (47),
(48) and (49), we have:

$\displaystyle\sqrt{n}\left\langle\frac{1}{2}\left[\check{\mathbf{u}}_{j}^{(1),[1]}+\check{\mathbf{u}}_{j}^{(1),[2]}-\left(\sqrt{1+b_{j}^{(1)[1]}}+\sqrt{1+b_{j}^{(1)[2]}}\right)\mathbf{u}_{j}^{(1)}\right],\mathbf{v}\right\rangle$
$\displaystyle=$
$\displaystyle\frac{1}{\sqrt{n}}\sum_{k=1}^{n}\eta_{jk}(\mathbf{v})+o_{p}(1)$
$\displaystyle\overset{d}{\to}$ $\displaystyle
N\left(0,\frac{1+\sigma_{j}^{2}}{\sigma_{j}^{4}}\|{\cal
P}_{j}^{\perp}\mathbf{v}\|^{2}\right).$

Recall that $b_{j}^{(1)[h]}\overset{p}{\to}0$. By Slutsky’s Theorem,

$\displaystyle\sqrt{n}\left((1+b_{j}^{(1)})\langle\check{\mathbf{u}}^{(1)}_{j},\mathbf{v}\rangle-\langle\mathbf{u}^{(1)}_{j},\mathbf{v}\rangle\right)$
$\displaystyle=$
$\displaystyle\frac{1}{\sqrt{1+b_{j}^{(1)[1]}}+\sqrt{1+b_{j}^{(1)[2]}}}\cdot\sqrt{n}\left\langle\frac{1}{2}\left[\check{\mathbf{u}}_{j}^{(1),[1]}+\check{\mathbf{u}}_{j}^{(1),[2]}-\left(\sqrt{1+b_{j}^{(1)[1]}}+\sqrt{1+b_{j}^{(1)[2]}}\right)\mathbf{u}_{j}^{(1)}\right],\mathbf{v}\right\rangle$
$\displaystyle\overset{d}{\to}$ $\displaystyle
N\left(0,\frac{1+\sigma_{j}^{2}}{\sigma_{j}^{4}}\|{\cal
P}_{j}^{\perp}\mathbf{v}\|^{2}\right).$

The claim (40) then follows.

We shall now prove (50). Note that ${\cal
P}_{j}\mathbf{u}_{j}^{(1)}=\mathbf{u}_{j}^{(1)}$, and ${\cal
C}_{j}=\frac{1}{\sigma_{j}^{2}}{\cal P}_{j}^{\perp}$, we have

$\displaystyle\eta_{jk}(\mathbf{v})=\frac{1}{\sigma_{j}^{2}}\left(\sigma_{j}\theta_{jk}+\mathscr{E}_{k}\times_{1}\mathbf{u}_{j}^{(1)}\times_{2}\mathbf{u}_{j}^{(2)}\dots\times_{p}\mathbf{u}_{j}^{(p)}\right)\left(\mathscr{E}_{k}\times_{1}({\cal
P}_{j}^{\perp}\mathbf{v})\times_{2}\mathbf{u}_{j}^{(2)}\dots\times_{p}\mathbf{u}_{j}^{(p)}\right).$

Both
$\sigma_{j}\theta_{jk}+\mathscr{E}_{k}\times_{1}\mathbf{u}_{j}^{(1)}\times_{2}\mathbf{u}_{j}^{(2)}\dots\times_{p}\mathbf{u}_{j}^{(p)}$
and $\mathscr{E}_{k}\times_{1}(\mathbb{I}_{d_{1}}{\cal
P}_{j}^{\perp}\mathbf{v})\times_{2}\mathbf{u}_{j}^{(2)}\dots\times_{p}\mathbf{u}_{j}^{(p)}$
are centered Gaussian random variables. Moreover, they are independent since
$\theta_{jk}$ is independent with $\mathscr{E}_{k}$, and
$\mathbf{u}_{j}^{(1)}\perp{\cal P}_{j}^{\perp}\mathbf{v}$. So

$\displaystyle\mathbb{E}\eta_{jk}(\mathbf{v})=0.$

More generally, Gaussian variable
$\sigma_{j}\theta_{jk}+\mathscr{E}_{k}\times_{1}\mathbf{u}_{j}^{(1)}\times_{2}\mathbf{u}_{j}^{(2)}\dots\times_{p}\mathbf{u}_{j}^{(p)}$
and Gaussian vector $\mathscr{E}_{k}\times_{1}({\cal
P}_{j}^{\perp})\times_{2}\mathbf{u}_{j}^{(2)}\dots\times_{p}\mathbf{u}_{j}^{(p)}$
are independent for the exact same reason. So direct calculation gives:

$\displaystyle{\rm var}(\eta_{jk}(\mathbf{v}))$ $\displaystyle=$
$\displaystyle\frac{1}{\sigma_{j}^{4}}\mathbb{E}\left(\sigma_{j}\theta_{jk}+\mathscr{E}_{k}\times_{1}\mathbf{u}_{j}^{(1)}\times_{2}\mathbf{u}_{j}^{(2)}\dots\times_{p}\mathbf{u}_{j}^{(p)}\right)^{2}$
$\displaystyle\mathbb{E}\bigg{[}\left(\mathscr{E}_{k}\times_{1}({\cal
P}_{j}^{\perp}\mathbf{v})\times_{2}\mathbf{u}_{j}^{(2)}\dots\times_{p}\mathbf{u}_{j}^{(p)}\right)^{2}\bigg{]}$
$\displaystyle=$ $\displaystyle\frac{1+\sigma_{j}^{2}}{\sigma_{j}^{4}}\|{\cal
P}_{j}^{\perp}\mathbf{v}\|^{2}.$

With the fact that $\eta_{jk}(\mathbf{v})$ are i.i.d for $k=1,2,\dots,n$, to
finish this part of the proof with CLT, it remains to check the Lindeberg
condition for CLT, which reduced to

$\displaystyle\frac{\mathbb{E}\eta_{jk}(\mathbf{v})^{2}\mathbb{I}\left(|\eta_{jk}(\mathbf{v})|\geq\tau\sqrt{n}\mathbb{E}^{1/2}\eta_{jk}(\mathbf{v})^{2}\right)}{\mathbb{E}\eta_{jk}(\mathbf{v})^{2}}\to
0\ {\rm as}\ n\to\infty$

for all $\tau>0$. Note that

$\displaystyle\frac{\mathbb{E}\eta_{jk}(\mathbf{v})^{2}\mathbb{I}\left(|\eta_{jk}(\mathbf{v})|\geq\tau\sqrt{n}\mathbb{E}^{1/2}\eta_{jk}(\mathbf{v})^{2}\right)}{\mathbb{E}\eta_{jk}(\mathbf{v})^{2}}\leq\frac{\mathbb{E}\eta_{jk}(\mathbf{v})^{4}}{\tau^{2}n\left(\mathbb{E}\eta_{jk}(\mathbf{v})^{2}\right)^{2}}.$

Since

$\displaystyle\mathbb{E}\eta_{jk}(\mathbf{v})^{4}$ $\displaystyle=$
$\displaystyle\mathbb{E}\left(\sigma_{j}\theta_{jk}+\mathscr{E}_{k}\times_{1}\mathbf{u}_{j}^{(1)}\times_{2}\mathbf{u}_{j}^{(2)}\dots\times_{p}\mathbf{u}_{j}^{(p)}\right)^{4}\mathbb{E}\left(\mathscr{E}_{k}\times_{1}({\cal
P}_{j}^{\perp}\mathbf{v})\times_{2}\mathbf{u}_{j}^{(2)}\dots\times_{p}\mathbf{u}_{j}^{(p)}\right)^{4},$

and

$\displaystyle\left(\mathbb{E}\eta_{jk}(\mathbf{v})^{2}\right)^{2}$
$\displaystyle=$
$\displaystyle\mathbb{E}^{2}\left(\sigma_{j}\theta_{jk}+\mathscr{E}_{k}\times_{1}\mathbf{u}_{j}^{(1)}\times_{2}\mathbf{u}_{j}^{(2)}\dots\times_{p}\mathbf{u}_{j}^{(p)}\right)^{2}\mathbb{E}^{2}\left(\mathscr{E}_{k}\times_{1}({\cal
P}_{j}^{\perp}\mathbf{v})\times_{2}\mathbf{u}_{j}^{(2)}\dots\times_{p}\mathbf{u}_{j}^{(p)}\right)^{2},$

with the fact that for a centered normal random variable $\xi$,
$\mathbb{E}\xi^{4}=3\mathbb{E}^{2}\xi^{2}$, we get

$\displaystyle\frac{\mathbb{E}\eta_{jk}(\mathbf{v})^{4}}{\tau^{2}n\left(\mathbb{E}\eta_{jk}(\mathbf{v})^{2}\right)^{2}}=\frac{1}{\tau^{2}n}\to
0,$

and (50) follows.

###  D.2 Step 2.

Again, it suffices to consider the case $q=1$. We shall now argue that

$\displaystyle\langle\check{\mathbf{u}}_{j}^{(1)},\mathbf{u}_{j}^{(1)}\rangle=\frac{1}{\sqrt{1+\frac{d_{1}}{n}\left(\frac{\sigma_{0}^{2}}{\sigma_{j}^{2}}+\frac{\sigma_{0}^{4}}{\sigma_{j}^{4}}\right)}}+O_{p}\left(\frac{d^{3/2}}{n^{3/2}}\right).$
(51)

Write $\tilde{\Sigma}^{[1]}=\widehat{\Sigma}^{[1]}-\mathscr{I}$,
$\tilde{\Sigma}^{[2]}=\widehat{\Sigma}^{[2]}-\mathscr{I}$, and let

$\displaystyle\grave{\mathbf{u}}_{j}^{(1),[1]}=\tilde{\Sigma}^{[1]}(\cdot,\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]},\check{\mathbf{u}}_{j}^{(1),[1]},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]}),$

and

$\displaystyle\grave{\mathbf{u}}_{j}^{(1),[2]}=\tilde{\Sigma}^{[2]}(\cdot,\widehat{\mathbf{u}}_{j}^{(2)[1]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[1]},\check{\mathbf{u}}_{j}^{(1),[2]},\widehat{\mathbf{u}}_{j}^{(2)[1]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[1]}).$

Since $\check{\mathbf{u}}_{j}^{(1),[1]}$ is the leading eigenvector of

$\widehat{\Sigma}^{[1]}(\cdot,\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]},\cdot,\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]}),$

it is also the leading eigenvector of

$\tilde{\Sigma}^{[1]}(\cdot,\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]},\cdot,\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]}),$

so

$\check{\mathbf{u}}_{j}^{(1),[1]}=\frac{\grave{\mathbf{u}}_{j}^{(1),[1]}}{\|\grave{\mathbf{u}}_{j}^{(1),[1]}\|},$

and similarly

$\check{\mathbf{u}}_{j}^{(1),[2]}=\frac{\grave{\mathbf{u}}_{j}^{(1),[2]}}{\|\grave{\mathbf{u}}_{j}^{(1),[2]}\|}.$

Write

$\mathbf{z}_{j}^{(1)[1]}:=\grave{\mathbf{u}}_{j}^{(1),[1]}-\langle\grave{\mathbf{u}}_{j}^{(1),[1]},\mathbf{u}_{j}^{(1)}\rangle\mathbf{u}_{j}^{(1)}-\mathbf{y}_{j}^{(1)[1]},$

where

$\mathbf{y}_{j}^{(1)[1]}:={\cal
P}_{\mathbf{u}_{j}^{(1)}}^{\perp}\left[\frac{2}{n}\sum_{k=1}^{n/2}\mathscr{E}_{k}(\cdot,\mathbf{u}_{j}^{(2)},\dots,\mathbf{u}_{j}^{(p)})\sigma_{j}\theta_{jk}+\frac{2}{n}\sum_{k=1}^{n/2}\mathscr{E}_{k}(\cdot,\mathbf{u}_{j}^{(2)},\dots,\mathbf{u}_{j}^{(p)})\mathscr{E}_{k}(\mathbf{u}_{j}^{(1)},\dots,\mathbf{u}_{j}^{(p)})\right].$
(52)

$\mathbf{z}_{j}^{(1)[2]}$ and $\mathbf{y}_{j}^{(1)[2]}$ are defined similarly
so that

$\grave{\mathbf{u}}_{j}^{(1),[1]}=\langle\grave{\mathbf{u}}_{j}^{(1),[1]},\mathbf{u}_{j}^{(1)}\rangle\mathbf{u}_{j}^{(1)}+\mathbf{y}_{j}^{(1)[1]}+\mathbf{z}_{j}^{(1)[1]},$
$\grave{\mathbf{u}}_{j}^{(1),[2]}=\langle\grave{\mathbf{u}}_{j}^{(1),[2]},\mathbf{u}_{j}^{(1)}\rangle\mathbf{u}_{j}^{(1)}+\mathbf{y}_{j}^{(1)[2]}+\mathbf{z}_{j}^{(1)[2]}.$

We shall treat the three terms in $\grave{\mathbf{u}}_{j}^{(1),[1]}$ and
$\grave{\mathbf{u}}_{j}^{(1),[1]}$ separately to show that

$\displaystyle\langle\grave{\mathbf{u}}_{j}^{(1),[1]},\mathbf{u}_{j}^{(1)}\rangle=\sigma_{j}^{2}+O_{p}\left(\frac{d}{n}\right),\
\langle\grave{\mathbf{u}}_{j}^{(1),[2]},\mathbf{u}_{j}^{(1)}\rangle=\sigma_{j}^{2}+O_{p}\left(\frac{d}{n}\right),$
(53)
$\displaystyle\left\|\mathbf{y}_{j}^{(1)[1]}+\mathbf{y}_{j}^{(1)[2]}\right\|=2\sqrt{\frac{d_{1}}{n}}\sqrt{\sigma_{0}^{2}\sigma_{j}^{2}+\sigma_{0}^{4}}+O_{p}\left(\frac{\sqrt{d}}{n}\right)+O_{p}\left(\frac{1}{\sqrt{n}}\right),$
(54) $\displaystyle\left\|\mathbf{y}_{j}^{(1)[1]}\right\|=$
$\displaystyle\sqrt{\frac{2d_{1}}{n}}\sqrt{\sigma_{0}^{2}\sigma_{j}^{2}+\sigma_{0}^{4}}+O_{p}\left(\frac{\sqrt{d}}{n}\right)+O_{p}\left(\frac{1}{\sqrt{n}}\right),$
(55) $\displaystyle\left\|\mathbf{y}_{j}^{(1)[2]}\right\|=$
$\displaystyle\sqrt{\frac{2d_{1}}{n}}\sqrt{\sigma_{0}^{2}\sigma_{j}^{2}+\sigma_{0}^{4}}+O_{p}\left(\frac{\sqrt{d}}{n}\right)+O_{p}\left(\frac{1}{\sqrt{n}}\right),$

and

$\displaystyle\mathbf{z}_{j}^{(1)[1]}=O_{p}\left(\frac{d}{n}\right),\
\mathbf{z}_{j}^{(1)[2]}=O_{p}\left(\frac{d}{n}\right).$ (56)

We first show that equation (51) follows from the bounds given in (53)-(56).
We start with $\|\grave{\mathbf{u}}_{j}^{(1),[1]}\|$. Combining (53)-(56), we
have

$\displaystyle\|\grave{\mathbf{u}}_{j}^{(1),[1]}\|$ $\displaystyle=$
$\displaystyle\sqrt{\left\langle\grave{\mathbf{u}}_{j}^{(1),[1]},\mathbf{u}_{j}^{(1)}\right\rangle^{2}+\left\|\mathbf{y}_{j}^{(1)[1]}+\mathbf{z}_{j}^{(1)[1]}\right\|^{2}}$
$\displaystyle=$
$\displaystyle\sqrt{\left[\sigma_{j}^{2}+O_{p}\left(\frac{d}{n}\right)\right]^{2}+\left[O_{p}\left(\sqrt{\frac{d}{n}}\right)+O_{p}\left(\frac{d}{n}\right)\right]^{2}}$
$\displaystyle=$ $\displaystyle\sigma_{j}^{2}+O_{p}\left(\frac{d}{n}\right),$
(57)

where the second equality follows from (53), (55), (56), and the last equality
holds because $d=o(n)$. Similarly we can derive that

$\|\grave{\mathbf{u}}_{j}^{(1),[2]}\|=\sigma_{j}^{2}+O_{p}\left({d}/{n}\right).$

Hence,

$\displaystyle\left\|\frac{\mathbf{y}_{j}^{(1)[1]}}{\|\grave{\mathbf{u}}_{j}^{(1),[1]}\|}+\frac{\mathbf{y}_{j}^{(1)[2]}}{\|\grave{\mathbf{u}}_{j}^{(1),[2]}\|}\right\|$
$\displaystyle=$
$\displaystyle\left\|\frac{1}{\sigma_{j}^{2}}(\mathbf{y}_{j}^{(1)[1]}+\mathbf{y}_{j}^{(1)[2]})+\left(\frac{1}{\|\grave{\mathbf{u}}_{j}^{(1),[1]}\|}-\frac{1}{\sigma_{j}^{2}}\right)\mathbf{y}_{j}^{(1)[1]}+\left(\frac{1}{\|\grave{\mathbf{u}}_{j}^{(1),[2]}\|}-\frac{1}{\sigma_{j}^{2}}\right)\mathbf{y}_{j}^{(1)[2]}\right\|$
$\displaystyle=$ $\displaystyle
2\sqrt{\frac{d_{1}}{n}\left(\frac{\sigma_{0}^{2}}{\sigma_{j}^{2}}+\frac{\sigma_{0}^{4}}{\sigma_{j}^{4}}\right)}+O_{p}\left(\frac{\sqrt{d}}{n}\right)+O_{p}\left(\frac{1}{\sqrt{n}}\right)+O_{p}\left(\frac{\sqrt{d}}{n}\right)\cdot
O_{p}\left(\frac{1}{\sqrt{n}}\right)$ $\displaystyle=$ $\displaystyle
2\sqrt{\frac{d_{1}}{n}\left(\frac{\sigma_{0}^{2}}{\sigma_{j}^{2}}+\frac{\sigma_{0}^{4}}{\sigma_{j}^{4}}\right)}+O_{p}\left(\frac{\sqrt{d}}{n}\right)+O_{p}\left(\frac{1}{\sqrt{n}}\right),$
(58)

where the second equality follows from (54), (55) and (57).

Moreover, (56), (57) imply that

$\displaystyle\left\|\frac{\mathbf{z}_{j}^{(1)[1]}}{\|\grave{\mathbf{u}}_{j}^{(1),[1]}\|}+\frac{\mathbf{z}_{j}^{(1)[2]}}{\|\grave{\mathbf{u}}_{j}^{(1),[2]}\|}\right\|=O_{p}\left(\frac{d}{n}\right),$
(59)

and (53) and (57) imply that

$\displaystyle\frac{\langle\grave{\mathbf{u}}_{j}^{(1),[1]},\mathbf{u}_{j}^{(1)}\rangle}{\|\grave{\mathbf{u}}_{j}^{(1),[1]}\|}+\frac{\langle\grave{\mathbf{u}}_{j}^{(1),[2]},\mathbf{u}_{j}^{(1)}\rangle}{\|\grave{\mathbf{u}}_{j}^{(1),[2]}\|}=2+O_{p}\left(\frac{d}{n}\right).$
(60)

Therefore,

$\displaystyle\langle\check{\mathbf{u}}_{j}^{(1)},\mathbf{u}_{j}^{(1)}\rangle$
$\displaystyle=$
$\displaystyle{\frac{\langle\grave{\mathbf{u}}_{j}^{(1),[1]},\mathbf{u}_{j}^{(1)}\rangle}{\|\grave{\mathbf{u}}_{j}^{(1),[1]}\|}+\frac{\langle\grave{\mathbf{u}}_{j}^{(1),[2]},\mathbf{u}_{j}^{(1)}\rangle}{\|\grave{\mathbf{u}}_{j}^{(1),[2]}\|}}\over\sqrt{\left(\frac{\langle\grave{\mathbf{u}}_{j}^{(1),[1]},\mathbf{u}_{j}^{(1)}\rangle}{\|\grave{\mathbf{u}}_{j}^{(1),[1]}\|}+\frac{\langle\grave{\mathbf{u}}_{j}^{(1),[2]},\mathbf{u}_{j}^{(1)}\rangle}{\|\grave{\mathbf{u}}_{j}^{(1),[2]}\|}\right)^{2}+\left(\frac{\mathbf{y}_{j}^{(1)[1]}}{\|\grave{\mathbf{u}}_{j}^{(1),[1]}\|}+\frac{\mathbf{y}_{j}^{(1)[2]}}{\|\grave{\mathbf{u}}_{j}^{(1),[2]}\|}+\frac{\mathbf{z}_{j}^{(1)[1]}}{\|\grave{\mathbf{u}}_{j}^{(1),[1]}\|}+\frac{\mathbf{z}_{j}^{(1)[2]}}{\|\grave{\mathbf{u}}_{j}^{(1),[2]}\|}\right)^{2}}$
$\displaystyle=$ $\displaystyle
1\Bigg{/}\sqrt{1+\left(\frac{\frac{\mathbf{y}_{j}^{(1)[1]}}{\|\grave{\mathbf{u}}_{j}^{(1),[1]}\|}+\frac{\mathbf{y}_{j}^{(1)[2]}}{\|\grave{\mathbf{u}}_{j}^{(1),[2]}\|}+\frac{\mathbf{z}_{j}^{(1)[1]}}{\|\grave{\mathbf{u}}_{j}^{(1),[1]}\|}+\frac{\mathbf{z}_{j}^{(1)[2]}}{\|\grave{\mathbf{u}}_{j}^{(1),[2]}\|}}{\frac{\langle\grave{\mathbf{u}}_{j}^{(1),[1]},\mathbf{u}_{j}^{(1)}\rangle}{\|\grave{\mathbf{u}}_{j}^{(1),[1]}\|}+\frac{\langle\grave{\mathbf{u}}_{j}^{(1),[2]},\mathbf{u}_{j}^{(1)}\rangle}{\|\grave{\mathbf{u}}_{j}^{(1),[2]}\|}}\right)^{2}}$
$\displaystyle=$ $\displaystyle
1\Bigg{/}\sqrt{1+\left(\frac{2\sqrt{\frac{d_{1}}{n}\left(\frac{\sigma_{0}^{2}}{\sigma_{j}^{2}}+\frac{\sigma_{0}^{4}}{\sigma_{j}^{4}}\right)}+O_{p}\left(\frac{\sqrt{d}}{n}\right)+O_{p}\left(\frac{1}{\sqrt{n}}\right)+O_{p}\left(\frac{d}{n}\right)}{2+O_{p}\left(\frac{d}{n}\right)}\right)^{2}}$
$\displaystyle=$ $\displaystyle
1\Bigg{/}\sqrt{1+\left\\{\left[\sqrt{\frac{d_{1}}{n}\left(\frac{\sigma_{0}^{2}}{\sigma_{j}^{2}}+\frac{\sigma_{0}^{4}}{\sigma_{j}^{4}}\right)}+O_{p}\left(\frac{1}{\sqrt{n}}\right)+O_{p}\left(\frac{d}{n}\right)\right]\cdot\left[1+O_{p}\left(\frac{d}{n}\right)\right]\right\\}^{2}}$
$\displaystyle=$ $\displaystyle
1\over\sqrt{1+\left[\sqrt{\frac{d_{1}}{n}\left(\frac{\sigma_{0}^{2}}{\sigma_{j}^{2}}+\frac{\sigma_{0}^{4}}{\sigma_{j}^{4}}\right)}+O_{p}\left(\frac{d}{n}\right)\right]^{2}}$
$\displaystyle=$
$\displaystyle\frac{1}{\sqrt{1+\frac{d_{1}}{n}\left(\frac{\sigma_{0}^{2}}{\sigma_{j}^{2}}+\frac{\sigma_{0}^{4}}{\sigma_{j}^{4}}\right)}}+O_{p}\left(\frac{d^{3/2}}{n^{3/2}}\right),$

where the third equation follows from (58)-(60), and the last equation follows
from the simple fact that

$\left|\frac{1}{\sqrt{1+(x+y)^{2}}}-\frac{1}{\sqrt{1+x^{2}}}\right|\leq\left|xy+\frac{1}{2}y^{2}\right|.$

It now remains to show (53)-(56).

####  D.2.1 Equations (54) and (55).

Write

$\mathbf{x}_{k}:=\mathscr{E}_{k}(\cdot,\mathbf{u}_{j}^{(2)},\dots,\mathbf{u}_{j}^{(p)})\in\mathbb{R}^{d_{1}}.$

If we choose an orthogonal basis of $\mathbb{R}^{d_{1}}$ with
$\mathbf{e}_{1}=\mathbf{u}_{j}^{(1)}$, then

$\displaystyle\left\|\frac{1}{2}(\mathbf{y}_{j}^{(1)[1]}+\mathbf{y}_{j}^{(1)[2]})\right\|=$
$\displaystyle\left\|{\cal
P}_{\mathbf{u}_{j}^{(1)}}^{\perp}\left[\frac{1}{n}\sum_{k=1}^{n}\mathbf{x}_{k}\big{(}\sigma_{j}\theta_{jk}+\mathbf{x}_{1,k}\big{)}\right]\right\|=\left\|\frac{1}{n}\sum_{k=1}^{n}\mathbf{x}_{(-1),k}\big{(}\sigma_{j}\theta_{jk}+\mathbf{x}_{1,k}\big{)}\right\|,$

in which $\mathbf{x}_{1,k}$ means the first entry of $\mathbf{x}_{k}$, while
$\mathbf{x}_{(-1),k}$ stands for all the other entries (having a dimension of
$d_{1}-1$).

Observe that $\mathbf{x}_{(-1),k}$ and
$\big{(}\sigma_{j}\theta_{jk}+\mathbf{x}_{1,k}\big{)}$ two independent group
of i.i.d. random variables, $\mathbf{x}_{(-1),k}$ follows distribution
$N(0,\sigma_{0}^{2}I_{d_{1}-1})$, and
$\big{(}\sigma_{j}\theta_{jk}+\mathbf{x}_{1,k}\big{)}$ follows distribution
$N(0,\sigma_{0}^{2}+\sigma_{j}^{2})$. Thus,

$\displaystyle\left\|\frac{1}{2}(\mathbf{y}_{j}^{(1)[1]}+\mathbf{y}_{j}^{(1)[2]})\right\|=$
$\displaystyle\sqrt{\frac{d_{1}-1}{n}}\sqrt{\sigma_{0}^{2}\sigma_{j}^{2}+\sigma_{0}^{4}}+O_{p}\left(\frac{\sqrt{d_{1}-1}}{n}\right)+O_{p}\left(\frac{1}{\sqrt{n}}\right).$

(55) can be proven in a similar way.

####  D.2.2 Equation (56).

Recall that

$\displaystyle\tilde{\Sigma}(\cdot,\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]},\grave{\mathbf{u}}_{j}^{(1),[1]},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})$
$\displaystyle=$
$\displaystyle\sum_{l=1}^{r}\sigma_{l}^{2}\left(\frac{2}{n}\sum_{k=1}^{n/2}\theta_{lk}^{2}\right)\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)^{2}\langle\mathbf{u}_{l}^{(1)},\grave{\mathbf{u}}_{j}^{(1),[1]}\rangle\mathbf{u}_{l}^{(1)}$
$\displaystyle+\sum_{l_{1}\neq
l_{2}}\sigma_{l_{1}}\sigma_{l_{2}}\left(\frac{2}{n}\sum_{k=1}^{n/2}\theta_{l_{1}k}\theta_{l_{2}k}\right)\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l_{1}}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l_{2}}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\langle\mathbf{u}_{l_{2}}^{(1)},\grave{\mathbf{u}}_{j}^{(1),[1]}\rangle\mathbf{u}_{l_{1}}^{(1)}$
$\displaystyle+\frac{2}{n}\sum_{k=1}^{n/2}\sum_{l=1}^{r}\sigma_{l}\theta_{lk}\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\mathscr{E}_{k}(\grave{\mathbf{u}}_{j}^{(1),[1]},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})\mathbf{u}_{l}^{(1)}$
$\displaystyle+\frac{2}{n}\sum_{k=1}^{n/2}\sum_{l=1}^{r}\mathscr{E}_{k}(\cdot,\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})\sigma_{l}\theta_{lk}\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\langle\mathbf{u}_{l}^{(1)},\grave{\mathbf{u}}_{j}^{(1),[1]}\rangle$
$\displaystyle+\frac{2}{n}\sum_{k=1}^{n/2}\mathscr{E}_{k}(\cdot,\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})\mathscr{E}_{k}(\grave{\mathbf{u}}_{j}^{(1),[1]},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})-\grave{\mathbf{u}}_{j}^{(1),[1]},$

so by definition,

$\displaystyle\mathbf{z}_{j}^{(1)[1]}=$ $\displaystyle\sum_{l\neq
j,l\in[r]}\sigma_{l}^{2}\left(\frac{2}{n}\sum_{k=1}^{n/2}\theta_{lk}^{2}\right)\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)^{2}\langle\mathbf{u}_{l}^{(1)},\grave{\mathbf{u}}_{j}^{(1),[1]}\rangle\mathbf{u}_{l}^{(1)}$
$\displaystyle+\sum_{l_{1}\neq l_{2},l_{1}\neq
j}\sigma_{l_{1}}\sigma_{l_{2}}\left(\frac{2}{n}\sum_{k=1}^{n/2}\theta_{l_{1}k}\theta_{l_{2}k}\right)\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l_{1}}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l_{2}}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\langle\mathbf{u}_{l_{2}}^{(1)},\grave{\mathbf{u}}_{j}^{(1),[1]}\rangle\mathbf{u}_{l_{1}}^{(1)}$
$\displaystyle+\frac{2}{n}\sum_{k=1}^{n/2}\sum_{l\neq
j,l\in[r]}\sigma_{l}\theta_{lk}\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\mathscr{E}_{k}(\grave{\mathbf{u}}_{j}^{(1),[1]},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})\mathbf{u}_{l}^{(1)}$
$\displaystyle+\frac{2}{n}\sum_{k=1}^{n/2}\sum_{l\neq
j,l\in[r]}\mathscr{E}_{k}(\cdot,\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})\sigma_{l}\theta_{lk}\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\langle\mathbf{u}_{l}^{(1)},\grave{\mathbf{u}}_{j}^{(1),[1]}\rangle$
$\displaystyle+{\cal
P}_{\mathbf{u}_{j}^{(1)}}^{\perp}\bigg{[}\frac{2}{n}\sum_{k=1}^{n/2}\mathscr{E}_{k}(\cdot,\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})\sigma_{j}\theta_{jk}\left(\prod_{q=2}^{p}\langle\mathbf{u}_{j}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\langle\mathbf{u}_{j}^{(1)},\grave{\mathbf{u}}_{j}^{(1),[1]}\rangle$
$\displaystyle\qquad\qquad-\frac{2}{n}\sum_{k=1}^{n/2}\mathscr{E}_{k}(\cdot,\mathbf{u}_{j}^{(2)},\dots,\mathbf{u}_{j}^{(p)})\sigma_{j}\theta_{jk}\bigg{]}$
$\displaystyle+{\cal
P}_{\mathbf{u}_{j}^{(1)}}^{\perp}\bigg{[}\frac{2}{n}\sum_{k=1}^{n/2}\mathscr{E}_{k}(\cdot,\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})\mathscr{E}_{k}(\grave{\mathbf{u}}_{j}^{(1),[1]},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})-\grave{\mathbf{u}}_{j}^{(1),[1]}$
$\displaystyle\qquad\qquad-\frac{2}{n}\sum_{k=1}^{n/2}\mathscr{E}_{k}(\cdot,\mathbf{u}_{j}^{(2)},\dots,\mathbf{u}_{j}^{(p)})\mathscr{E}_{k}(\mathbf{u}_{j}^{(1)},\dots,\mathbf{u}_{j}^{(p)})+\mathbf{u}_{j}^{(1)}\bigg{]},$
(61)

and we will deal with those six terms one by one.

##### The first term.

$\displaystyle\mathbf{z}_{1}^{[1]}:=\sum_{l\neq
j,l\in[r]}\sigma_{l}^{2}\left(\frac{2}{n}\sum_{k=1}^{n/2}\theta_{lk}^{2}\right)\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)^{2}\langle\mathbf{u}_{l}^{(1)},\grave{\mathbf{u}}_{j}^{(1),[1]}\rangle\mathbf{u}_{l}^{(1)}.$

Observe that

$\displaystyle\left\|\mathbf{z}_{1}^{[1]}\right\|\leq\big{(}\delta^{(1)[2]}\big{)}^{2p-1}\sum_{l\neq
j,l\in[r]}\sigma_{l}^{2}\left|\frac{2}{n}\sum_{k=1}^{n/2}\theta_{lk}^{2}\right|=O_{p}\left[\left(\sqrt{\frac{d}{n}}\right)^{2p-1}\right]=O_{p}\left[\left(\frac{d}{n}\right)^{3/2}\right].$
(62)

The last two equality’s follow from (44), and $p\geq 2$. Because of the
assumption $d=o(n)$, it implies

$\left\|\mathbf{z}_{1}^{[1]}\right\|\leq O_{p}\left(\frac{d}{n}\right).$

##### The second term.

$\displaystyle\mathbf{z}_{2}^{[1]}:=$ $\displaystyle\sum_{l_{1}\neq
l_{2},l_{1}\neq
j}\sigma_{l_{1}}\sigma_{l_{2}}\left(\frac{2}{n}\sum_{k=1}^{n/2}\theta_{l_{1}k}\theta_{l_{2}k}\right)\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l_{1}}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l_{2}}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\langle\mathbf{u}_{l_{2}}^{(1)},\grave{\mathbf{u}}_{j}^{(1),[1]}\rangle\mathbf{u}_{l_{1}}^{(1)}$
$\displaystyle=$ $\displaystyle\sum_{l_{1}\neq l_{2},l_{1}\neq j,l_{2}\neq
j}\sigma_{l_{1}}\sigma_{l_{2}}\left(\frac{2}{n}\sum_{k=1}^{n/2}\theta_{l_{1}k}\theta_{l_{2}k}\right)\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l_{1}}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l_{2}}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\langle\mathbf{u}_{l_{2}}^{(1)},\grave{\mathbf{u}}_{j}^{(1),[1]}\rangle\mathbf{u}_{l_{1}}^{(1)}$
$\displaystyle+\sum_{l_{1}\neq
j}\sigma_{l_{1}}\sigma_{j}\left(\frac{2}{n}\sum_{k=1}^{n/2}\theta_{l_{1}k}\theta_{jk}\right)\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l_{1}}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\left(\prod_{q=2}^{p}\langle\mathbf{u}_{j}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\langle\mathbf{u}_{j}^{(1)},\grave{\mathbf{u}}_{j}^{(1),[1]}\rangle\mathbf{u}_{l_{1}}^{(1)}.$

Because

$\left|\frac{2}{n}\sum_{k=1}^{n/2}\theta_{l_{1}k}\theta_{l_{2}k}\right|=O_{p}(\frac{1}{\sqrt{n}}),\
\forall l_{1}\neq l_{2},$

we immediately have

$\displaystyle\left\|\mathbf{z}_{2}^{[1]}\right\|$ $\displaystyle\leq$
$\displaystyle\big{(}\delta^{(1)[2]}\big{)}^{2p-1}\sum_{l_{1}\neq
l_{2},l_{1}\neq j,l_{2}\neq
j}\sigma_{l_{1}}\sigma_{l_{2}}\left|\frac{2}{n}\sum_{k=1}^{n/2}\theta_{l_{1}k}\theta_{l_{2}k}\right|+\big{(}\delta^{(1)[2]}\big{)}^{p-1}\sum_{l_{1}\neq
j}\sigma_{l_{1}}\sigma_{j}\left|\frac{2}{n}\sum_{k=1}^{n/2}\theta_{l_{1}k}\theta_{jk}\right|$
$\displaystyle=$ $\displaystyle O_{p}(\frac{1}{\sqrt{n}})\cdot
O_{p}\left[\left(\sqrt{\frac{d}{n}}\right)^{2p-1}\right]+O_{p}(\frac{1}{\sqrt{n}})\cdot\left[\left(\sqrt{\frac{d}{n}}\right)^{p-1}\right]$
$\displaystyle=$ $\displaystyle O_{p}\left(\frac{\sqrt{d}}{n}\right).$ (63)

##### The third term.

$\displaystyle\mathbf{z}_{3}^{[1]}:=\frac{2}{n}\sum_{k=1}^{n/2}\sum_{l\neq
j,l\in[r]}\sigma_{l}\theta_{lk}\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\mathscr{E}_{k}(\grave{\mathbf{u}}_{j}^{(1),[1]},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})\mathbf{u}_{l}^{(1)}.$

Observe that

$\displaystyle\left\|\mathbf{z}_{3}^{[1]}\right\|\leq$
$\displaystyle\sum_{l\neq
j,l\in[r]}\sigma_{l}\left|\prod_{q=2}^{p}\langle\mathbf{u}_{l}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right|\cdot\left|\frac{2}{n}\sum_{k=1}^{n/2}\theta_{lk}\mathscr{E}_{k}(\grave{\mathbf{u}}_{j}^{(1),[1]},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})\right|$
$\displaystyle\leq$ $\displaystyle
r\sigma_{1}(\delta^{(1)[2]})^{p-1}\cdot\|\widehat{\Sigma}_{\theta,\mathscr{E}}^{[1]}\|$
$\displaystyle=$ $\displaystyle
O_{p}\left(\sqrt{\frac{d}{n}}\right)^{p-1}\cdot
O_{p}\left(\sqrt{\frac{d}{n}}\right)$ $\displaystyle=$ $\displaystyle
O_{p}\left(\frac{d}{n}\right),$ (64)

in which the second to last equality follows from (44) and (43).

##### The fourth term.

$\displaystyle\mathbf{z}_{4}^{[1]}:=\frac{2}{n}\sum_{k=1}^{n/2}\sum_{l\neq
j,l\in[r]}\mathscr{E}_{k}(\cdot,\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})\sigma_{l}\theta_{lk}\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\langle\mathbf{u}_{l}^{(1)},\grave{\mathbf{u}}_{j}^{(1),[1]}\rangle.$

Similar to $\mathbf{z}_{3}^{[1]}$,

$\displaystyle\big{\|}\mathbf{z}_{4}^{[1]}\big{\|}\leq\|\widehat{\Sigma}_{\theta,\mathscr{E}}\|\cdot\left(\delta^{(1)[2]}\right)^{p}=O_{p}\left(\frac{d^{3/2}}{n^{3/2}}\right)=O_{p}\left(d\over
n\right).$

##### The fifth term.

$\displaystyle\mathbf{z}_{5}^{[1]}:=$ $\displaystyle{\cal
P}_{\mathbf{u}_{j}^{(1)}}^{\perp}\bigg{[}\frac{2}{n}\sum_{k=1}^{n/2}\mathscr{E}_{k}(\cdot,\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})\sigma_{j}\theta_{jk}\left(\prod_{q=2}^{p}\langle\mathbf{u}_{j}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\langle\mathbf{u}_{j}^{(1)},\grave{\mathbf{u}}_{j}^{(1),[1]}\rangle$
$\displaystyle\qquad\qquad-\frac{2}{n}\sum_{k=1}^{n/2}\mathscr{E}_{k}(\cdot,\mathbf{u}_{j}^{(2)},\dots,\mathbf{u}_{j}^{(p)})\sigma_{j}\theta_{jk}\bigg{]}.$

we shall introduce the notation $\biguplus$, as the operation of expanding all
the

$\widehat{\mathbf{u}}_{j}^{(q)[2]}=\mathbf{u}_{j}^{(q)}+\Delta_{j}^{(q)[2]},$

and

$\grave{\mathbf{u}}_{j}^{(1),[1]}=\mathbf{u}_{j}^{(1)}+\grave{\Delta}_{j}^{(1)[1]},$

and then keep all the terms with at least one $\Delta_{j}^{(\cdot)[2]}$ or
$\grave{\Delta}_{j}^{(1)[1]}$ in it. For instance, here, expanding

$\frac{2}{n}\sum_{k=1}^{n/2}\mathscr{E}_{k}(\cdot,\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})\sigma_{j}\theta_{jk}\left(\prod_{q=2}^{p}\langle\mathbf{u}_{j}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\langle\mathbf{u}_{j}^{(1)},\grave{\mathbf{u}}_{j}^{(1),[1]}\rangle$

would result in $2^{2p-1}$ terms, and we keep everything other than the term

$\frac{2}{n}\sum_{k=1}^{n/2}\mathscr{E}_{k}(\cdot,\mathbf{u}_{j}^{(2)},\dots,\mathbf{u}_{j}^{(p)})\sigma_{j}\theta_{jk}\left(\prod_{q=1}^{p}\langle\mathbf{u}_{j}^{(q)},\mathbf{u}_{j}^{(q)}\rangle\right).$

Then we have:

$\displaystyle\mathbf{z}_{5}^{[1]}=$ $\displaystyle{\cal
P}_{\mathbf{u}_{j}^{(1)}}^{\perp}\left\\{\biguplus\Bigg{[}\frac{2}{n}\sum_{k=1}^{n/2}\mathscr{E}_{k}(\cdot,\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})\sigma_{j}\theta_{jk}\left(\prod_{q=2}^{p}\langle\mathbf{u}_{j}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\langle\mathbf{u}_{j}^{(1)},\grave{\mathbf{u}}_{j}^{(1),[1]}\rangle\Bigg{]}\right\\}.$

Then,

$\displaystyle\big{\|}\mathbf{z}_{5}^{[1]}\big{\|}\leq\left\|\biguplus\Bigg{[}\frac{2}{n}\sum_{k=1}^{n/2}\mathscr{E}_{k}(\cdot,\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})\sigma_{j}\theta_{jk}\left(\prod_{q=1}^{p}\langle\mathbf{u}_{j}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\Bigg{]}\right\|,$

and for every term in $\biguplus$, its norm is bounded by

$\|\widehat{\Sigma}_{\theta,\mathscr{E}}\|\cdot\delta^{(1)[2]}=O_{p}\big{(}\frac{d}{n}\big{)},$

so that

$\displaystyle\big{\|}\mathbf{z}_{5}^{[1]}\big{\|}=O_{p}\left(\frac{d}{n}\right).$

##### The sixth term.

This term can be treated in a similar fashion as the last term.

$\displaystyle\mathbf{z}_{6}^{[1]}:=$ $\displaystyle{\cal
P}_{\mathbf{u}_{j}^{(1)}}^{\perp}\bigg{[}\frac{2}{n}\sum_{k=1}^{n/2}\mathscr{E}_{k}(\cdot,\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})\mathscr{E}_{k}(\grave{\mathbf{u}}_{j}^{(1),[1]},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})-\grave{\mathbf{u}}_{j}^{(1),[1]}$
$\displaystyle\qquad\qquad-\frac{2}{n}\sum_{k=1}^{n/2}\mathscr{E}_{k}(\cdot,\mathbf{u}_{j}^{(2)},\dots,\mathbf{u}_{j}^{(p)})\mathscr{E}_{k}(\mathbf{u}_{j}^{(1)},\dots,\mathbf{u}_{j}^{(p)})+\mathbf{u}_{j}^{(1)}\bigg{]}$
$\displaystyle=$ $\displaystyle{\cal
P}_{\mathbf{u}_{j}^{(1)}}^{\perp}\Bigg{[}\bigg{(}\frac{2}{n}\sum_{k=1}^{n/2}\mathscr{E}_{k}\otimes\mathscr{E}_{k}-\mathscr{I}\bigg{)}(\cdot,\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]},\grave{\mathbf{u}}_{j}^{(1),[1]},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})$
$\displaystyle\qquad\qquad-\bigg{(}\frac{2}{n}\sum_{k=1}^{n/2}\mathscr{E}_{k}\otimes\mathscr{E}_{k}-\mathscr{I}\bigg{)}(\cdot,\mathbf{u}_{j}^{(2)},\dots,\mathbf{u}_{j}^{(p)},\mathbf{u}_{j}^{(1)},\dots,\mathbf{u}_{j}^{(p)})\Bigg{]}$
$\displaystyle=$ $\displaystyle{\cal
P}_{\mathbf{u}_{j}^{(1)}}^{\perp}\left\\{\biguplus\Bigg{[}\bigg{(}\frac{2}{n}\sum_{k=1}^{n/2}\mathscr{E}_{k}\otimes\mathscr{E}_{k}-\mathscr{I}\bigg{)}(\cdot,\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]},\grave{\mathbf{u}}_{j}^{(1),[1]},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})\Bigg{]}\right\\},$

then,

$\displaystyle\big{\|}\mathbf{z}_{6}^{[1]}\big{\|}\leq\left\|\biguplus\Bigg{[}\bigg{(}\frac{2}{n}\sum_{k=1}^{n/2}\mathscr{E}_{k}\otimes\mathscr{E}_{k}-\mathscr{I}\bigg{)}(\cdot,\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]},\widehat{\mathbf{u}}_{j}^{(1)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})\Bigg{]}\right\|,$

and for every term in $\biguplus$, its norm is bounded by

$\|\widehat{\Sigma}_{\mathscr{E}}-\mathscr{I}\|\cdot\delta^{(1)[2]}=O_{p}\big{(}\frac{d}{n}\big{)},$

so that

$\displaystyle\big{\|}\mathbf{z}_{6}^{[1]}\big{\|}=O_{p}\left(\frac{d}{n}\right).$

####  D.2.3 Equation (53)

We now show that

$\langle\grave{\mathbf{u}}_{j}^{(1),[1]},\mathbf{u}_{j}^{(1)}\rangle=\sigma_{j}^{2}+O_{p}\left(\frac{d}{n}\right).$

It follows immediately, by symmetry, that

$\displaystyle\langle\grave{\mathbf{u}}_{j}^{(1),[2]},\mathbf{u}_{j}^{(1)}\rangle=\sigma_{j}^{2}+O_{p}\left(\frac{d}{n}\right).$

By definition,

$\displaystyle\langle\grave{\mathbf{u}}_{j}^{(1),[1]},\mathbf{u}_{j}^{(1)}\rangle$
$\displaystyle=$
$\displaystyle\sigma_{j}^{2}\left(\frac{2}{n}\sum_{k=1}^{n/2}\theta_{jk}^{2}\right)\left(\prod_{q=2}^{p}\langle\mathbf{u}_{j}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)^{2}\langle\mathbf{u}_{j}^{(1)},\grave{\mathbf{u}}_{j}^{(1),[1]}\rangle$
$\displaystyle+\sum_{l\neq
j}\sigma_{l}\left(\frac{2}{n}\sum_{k=1}^{n/2}\theta_{jk}\theta_{lk}\right)\left(\prod_{q=2}^{p}\langle\mathbf{u}_{j}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\langle\mathbf{u}_{l}^{(1)},\grave{\mathbf{u}}_{j}^{(1),[1]}\rangle$
$\displaystyle+\frac{2}{n}\sum_{k=1}^{n/2}\sigma_{j}\theta_{jk}\left(\prod_{q=2}^{p}\langle\mathbf{u}_{j}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\mathscr{E}_{k}(\grave{\mathbf{u}}_{j}^{(1),[1]},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})$
$\displaystyle+\frac{2}{n}\sum_{k=1}^{n/2}\sum_{l=1}^{r}\mathscr{E}_{k}(\mathbf{u}_{j}^{(1)},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})\sigma_{l}\theta_{lk}\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\langle\mathbf{u}_{j}^{(1)},\grave{\mathbf{u}}_{j}^{(1),[1]}\rangle$
$\displaystyle+\frac{2}{n}\sum_{k=1}^{n/2}(\mathscr{E}_{k}\otimes\mathscr{E}_{k}-\mathscr{I})(\mathbf{u}_{j}^{(1)},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]},\grave{\mathbf{u}}_{j}^{(1),[1]},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]}).$
(65)

All the terms except the first one will be bounded with similar techniques as
we bound the six terms in $\mathbf{z}_{j}^{(1)[1]}$, and we omit some of the
details.

##### The first term.

Observe that

$1-\langle\mathbf{u}_{j}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\leq\big{(}\delta^{(1)[2]}\big{)}^{2},\
1-\langle\mathbf{u}_{j}^{(1)},\grave{\mathbf{u}}_{j}^{(1)[1]}\rangle\leq\big{(}\delta^{(1)[2]}\big{)}^{2}.$

Thus,

$\sigma_{j}^{2}\left(\frac{2}{n}\sum_{k=1}^{n/2}\theta_{jk}^{2}\right)\left(\prod_{q=2}^{p}\langle\mathbf{u}_{j}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)^{2}\langle\mathbf{u}_{j}^{(1)},\grave{\mathbf{u}}_{j}^{(1),[1]}\rangle=\sigma_{j}^{2}+O_{p}\left(\frac{d}{n}\right).$

##### The second term.

$\displaystyle\sum_{l\neq
j}\sigma_{l}\left(\frac{2}{n}\sum_{k=1}^{n/2}\theta_{jk}\theta_{lk}\right)\left(\prod_{q=2}^{p}\langle\mathbf{u}_{j}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\langle\mathbf{u}_{l}^{(1)},\grave{\mathbf{u}}_{j}^{(1),[1]}\rangle$
$\displaystyle=$ $\displaystyle O_{p}(\frac{1}{\sqrt{n}})\cdot
O_{p}\left[\left(\sqrt{\frac{d}{n}}\right)^{2p-1}\right]$ $\displaystyle=$
$\displaystyle o_{p}(\frac{1}{\sqrt{n}}).$

##### The third term.

Observe that

$\displaystyle\frac{2}{n}\sum_{k=1}^{n/2}\sigma_{j}\theta_{jk}\left(\prod_{q=2}^{p}\langle\mathbf{u}_{j}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\mathscr{E}_{k}(\grave{\mathbf{u}}_{j}^{(1),[1]},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})$
$\displaystyle=$
$\displaystyle\frac{2}{n}\sum_{k=1}^{n/2}\sigma_{j}\theta_{jk}\left(\prod_{q=2}^{p}\langle\mathbf{u}_{j}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\mathscr{E}_{k}(\mathbf{u}_{j}^{(1),[1]},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})$
$\displaystyle+\frac{2}{n}\sum_{k=1}^{n/2}\sigma_{j}\theta_{jk}\left(\prod_{q=2}^{p}\langle\mathbf{u}_{j}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\mathscr{E}_{k}(\grave{\mathbf{u}}_{j}^{(1),[1]}-\mathbf{u}_{j}^{(1),[1]},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]}),$

in which the first term is $O_{p}(1/\sqrt{n})$ because
$\theta_{jk}\mathscr{E}_{k},k\in[n/2]$ are independent with
$\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]}$,
and the second term is bounded by

$\|\widehat{\Sigma}_{\theta,\mathscr{E}}\|\cdot\delta^{(1)[2]}=O_{p}\big{(}\frac{d}{n}\big{)},$

so

$\frac{2}{n}\sum_{k=1}^{n/2}\sigma_{j}\theta_{jk}\left(\prod_{q=2}^{p}\langle\mathbf{u}_{j}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\mathscr{E}_{k}(\grave{\mathbf{u}}_{j}^{(1),[1]},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})=O_{p}\left(d\over
n\right)+O_{p}\left(1\over\sqrt{n}\right)=O_{p}\left(d\over n\right).$

The last equality follows from the assumption $d=o(n)$.

##### The fourth term.

Recall that $\widehat{\mathbf{u}}_{j}^{(q)[2]}$ is a function of the second
half of the data, which is independent of the first half of the data, i.e.,
all the random variables with index $k\leq n/2$, so conditional on the second
half of the data, for any $l\in[r]$,

$\mathscr{E}_{k}(\mathbf{u}_{j}^{(1)},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})\theta_{lk}$

are a product of two independent $N(0,\sigma_{0}^{2})$ and $N(0,1)$ variables,
and they are i.i.d across $k\leq n/2$. So we have

$\frac{2}{n}\sum_{k=1}^{n/2}\sum_{l=1}^{r}\mathscr{E}_{k}(\mathbf{u}_{j}^{(1)},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]})\sigma_{l}\theta_{lk}\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)\langle\mathbf{u}_{j}^{(1)},\grave{\mathbf{u}}_{j}^{(1),[1]}\rangle=O_{p}(\frac{1}{\sqrt{n}}).$

##### The fifth term.

Similar to the sixth term in $\mathbf{z}_{j}^{(1)[1]}$,

$\frac{2}{n}\sum_{k=1}^{n/2}(\mathscr{E}_{k}\otimes\mathscr{E}_{k}-\mathscr{I})(\mathbf{u}_{j}^{(1)},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]},\grave{\mathbf{u}}_{j}^{(1),[1]},\widehat{\mathbf{u}}_{j}^{(2)[2]},\dots,\widehat{\mathbf{u}}_{j}^{(p)[2]}).$

Combining the five bounds above, we get

$\langle\grave{\mathbf{u}}_{j}^{(1),[1]},\mathbf{u}_{j}^{(1)}\rangle=\sigma_{j}^{2}+O_{p}\left(\frac{d}{n}\right)+O_{p}\left(1\over\sqrt{n}\right)=O_{p}\left(d\over
n\right).$

The last equality follows from the assumption $d=o(n)$.

###  D.3 Step 3.

Recall that

$\widehat{b}_{j}^{(q)[1]}:=\left\langle\widehat{\mathbf{u}}_{j}^{(q),[1][1]},\widehat{\mathbf{u}}_{j}^{(q),[1][2]}\right\rangle-1,$
$\widehat{b}_{j}^{(q)[2]}:=\left\langle\widehat{\mathbf{u}}_{j}^{(q),[2][1]},\widehat{\mathbf{u}}_{j}^{(q),[2][2]}\right\rangle-1.$

In this part, we use Lemma 8 to show that

$\displaystyle\sqrt{n}\left(\widehat{b}_{j}^{(1)[1]}-b_{j}^{(1)[1]}\right)\overset{p}{\to}0,$
(66)

in which $b_{j}^{(1)[1]}$ is defined in Lemma 3. Similarly
$\sqrt{n}\left(\widehat{b}_{j}^{(1)[2]}-b_{j}^{(1)[2]}\right)\overset{p}{\to}0$.
Then, recall that by definition (39),

$\displaystyle
b_{j}^{(1)}=\frac{\left\|\check{\mathbf{u}}_{j}^{(1),[1]}+\check{\mathbf{u}}_{j}^{(1),[2]}\right\|}{\sqrt{1+b_{j}^{(1)[1]}}+\sqrt{1+b_{j}^{(1)[2]}}}-1,$

and by definition (13),

$\displaystyle\widehat{b}_{j}^{(1)}=\frac{\left\|\check{\mathbf{u}}_{j}^{(q),[1]}+\check{\mathbf{u}}_{j}^{(q),[2]}\right\|}{\sqrt{1+\widehat{b}_{j}^{(1)[1]}}+\sqrt{1+\widehat{b}_{j}^{(1)[2]}}}-1,$

we have that (42) follows from (66) and the fact that
$\left\|\check{\mathbf{u}}_{j}^{(1),[1]}+\check{\mathbf{u}}_{j}^{(1),[2]}\right\|\leq
2$. Now we turn our attention to (66).

By Lemma 4,

$\displaystyle\bar{\mathbb{E}}\|\widehat{M}_{j}^{[1]}-M_{j}^{[1]}\|$
$\displaystyle\leq$ $\displaystyle
C\|M_{j}^{[1]}\|\left(\sqrt{\frac{r(M_{j}^{[1]})}{n}}\bigvee\frac{r(M_{j}^{[1]})}{n}\right)$
$\displaystyle\leq$
$\displaystyle\frac{C(\sigma_{j}^{2}+1)(\sigma_{j}^{2}+d_{1}+1)}{(\frac{1}{2}\sigma_{j}^{2}+1)n},$

on events $\mathcal{B}_{n}$. ($M_{j}^{[1]}$, $\widehat{M}_{j}^{[1]}$ and
events $\mathcal{B}_{n}$ are defined at the beginning of the proof for Lemma
3.) The last inequality holds because of (98) and (94). Remember that
$\bar{g}_{1}\geq\frac{1}{2}\sigma_{j}^{2}$ (inequality (96)), so with
assumption $d=o(n)$, we have that for large enough $n$,
$\bar{\mathbb{E}}\|\widehat{M}_{j}^{[1]}-M_{j}^{[1]}\|\leq\frac{1}{4}\bar{g}_{1}$,
i.e., if we let $\gamma=\frac{1}{4}$, then

$\displaystyle\bar{\mathbb{E}}\|\widehat{M}_{j}^{[1]}-M_{j}^{[1]}\|\leq\frac{(1-2\gamma)\bar{g}_{1}}{2}.$

$1+b_{j}^{(1)[1]}\geq 2\gamma$ is also satisfied for large enough $n$ by
inequality (99).

Let $t=\left(\frac{n}{2r(M_{j}^{[1]})}\right)^{\frac{1}{3}}$. First note that
on the event $\mathcal{B}_{n}$,

$\displaystyle
t\geq\left(\frac{n(\frac{1}{2}\sigma_{j}^{2}+1)}{2(\sigma_{j}^{2}+d_{1}+1)}\right)^{\frac{1}{3}},$

by inequality (98), with assumption $d=o(n)$, we have $t\geq 1$ for large
enough $n$.

Let $D:=D_{\frac{1}{4}}$ as in Lemma 8. We have

$\displaystyle D\|M_{j}^{[1]}\|\left(\sqrt{\frac{t}{n}}\bigvee\sqrt{\frac{\log
n}{n}}\right)\leq
D(\sigma_{j}^{2}+1)\left(n^{-\frac{2}{3}}\bigvee\sqrt{\frac{\log
n}{n}}\right)\leq\frac{1}{8}\bar{g}_{1}$

for large enough $n$.

So all the conditions in Lemma 8 are satisfied with $\widehat{M}_{j}^{[1]}$
and $M_{j}^{[1]}$, conditional on $\\{\mathscr{X}_{k},k=n/2+1,\dots,n\\}$
under events $\mathcal{B}_{n}$, for large enough $n$.

Observe that conditional on $\\{\mathscr{X}_{k},k=n/2+1,\dots,n\\}$,
$\widehat{b}_{j}^{(1)[1]}$ and $b_{j}^{(1)[1]}$ are just defined as the
$\widehat{b}_{1}$ and $b_{1}$ [defined in (78) and (79)] corresponding to
$M_{j}^{[1]}$. Now we can apply Lemma 8:

$\displaystyle\sqrt{n}\left|\widehat{b}_{j}^{(1)[1]}-b_{j}^{(1)[1]}\right|$
$\displaystyle\leq$ $\displaystyle
D\sqrt{n}\frac{\|M_{j}^{[1]}\|^{2}}{\bar{g}_{1}^{2}}\left(\sqrt{\frac{r(M_{j}^{[1]})}{n}}\bigvee\sqrt{\frac{t}{n}}\bigvee\sqrt{\frac{\log
n}{n}}\right)\sqrt{\frac{t}{n}}$ $\displaystyle\leq$ $\displaystyle
D\frac{\|M_{j}^{[1]}\|^{2}}{\bar{g}_{1}^{2}}\left(\left(\frac{r(M_{j}^{[1]})}{n}\right)^{\frac{1}{3}}\bigvee\frac{1}{r(M_{j}^{[1]})^{\frac{1}{3}}n^{\frac{1}{6}}}\bigvee\frac{\sqrt{\log
n}}{r(M_{j}^{[1]})^{\frac{1}{6}}n^{\frac{1}{3}}}\right)$ $\displaystyle\leq$
$\displaystyle
D\frac{\|M_{j}^{[1]}\|^{2}}{\bar{g}_{1}^{2}}\left(\left(\frac{r(M_{j}^{[1]})}{n}\right)^{\frac{1}{3}}\bigvee\frac{1}{n^{\frac{1}{6}}}\right)$
$\displaystyle\leq$ $\displaystyle
D\left(\frac{\sigma_{j}^{2}+1}{\frac{1}{2}\sigma_{j}^{2}}\right)^{2}\left[\left(\frac{\sigma_{j}^{2}+d_{1}+1}{n(\frac{1}{2}\sigma_{j}^{2}+1)}\right)^{\frac{1}{3}}\bigvee\frac{1}{n^{\frac{1}{6}}}\
\right]$ (67)

conditional on $\\{\mathscr{X}_{k},k=n/2+1,\dots,n\\}$ under events
$\mathcal{B}_{n}$, for large enough $n$, with probability at least $1-e^{-t}$.
The last inequality holds because of (94), (96) and (98).

Since on the event $\mathcal{B}_{n}$, conditional on
$\\{\mathscr{X}_{n/2+1},\dots,\mathscr{X}_{n}\\}$, (67) holds with probability
at least $1-e^{-t}\geq
1-\exp\left[-\left(\frac{n(\frac{1}{2}\sigma_{j}^{2}+1)}{2(\sigma_{j}^{2}+d_{1}+1)}\right)^{\frac{1}{3}}\right]$,
we have that on the event $\mathcal{B}_{n}$, (67) also holds with probability
at least
$1-\exp\left[-\left(\frac{n(\frac{1}{2}\sigma_{j}^{2}+1)}{2(\sigma_{j}^{2}+d_{1}+1)}\right)^{\frac{1}{3}}\right]$.
Combine with assumption $\frac{d}{n}\to 0$, we have that

$\displaystyle\sqrt{n}\left(\widehat{b}_{j}^{(1)[1]}-b_{j}^{(1)[1]}\right)\cdot\mathbb{I}(\mathcal{B}_{n})\overset{p}{\to}0,$

which leads to

$\displaystyle\sqrt{n}\left(\widehat{b}_{j}^{(1)[1]}-b_{j}^{(1)[1]}\right)\overset{p}{\to}0.$

##  Appendix E Proof of Lemma 3

The proof of Lemma 3 relies heavily on the techniques and results from
Koltchinskii and Lounici (2014) which we will review first.

###  E.1 Preliminaries

Let $\mathbb{H}$ be a Hilbert space and $M:\mathbb{H}\to\mathbb{H}$ be a
compact symmetric nonnegative definite operator. It is well known that the
following spectral representation holds

$\displaystyle M=\sum_{r>1}\mu_{r}{\cal P}_{r}$

with distinct non-zero eigenvalues $\mu_{r}$ arranged in decreasing order
$\mu_{1}>\mu_{2}>\dots\geq 0$, and ${\cal P}_{r}$ are the corresponding
spectral projectors. The effective rank of $M$ is defined as

$\displaystyle r(M):=\frac{{\rm tr}(M)}{\|M\|}.$

We will use in particular the results from Koltchinskii and Lounici (2014) for
the estimation of ${\cal P}_{1}$, in the case where

$\displaystyle{\cal P}_{1}=\mathbf{u}_{1}\otimes\mathbf{u}_{1},$

i.e., estimating the leading eigenvector in the case that the leading
eigenvalue is an isolated simple eigenvalue. Let $Y_{1},Y_{2},\dots,Y_{n}$ be
i.i.d. centered Gaussian random vectors in $\mathbb{R}^{m}$ with covariance
$M=\mathbb{E}(Y\otimes Y)$. Let

$\displaystyle\widehat{M}:=\frac{1}{n}\sum_{k=1}^{n}Y_{k}\otimes Y_{k}$

be the sample covariance matrix based on the observations
$(Y_{1},Y_{2},\dots,Y_{n})$. The following lemma is a restatement of Theorem 1
from Koltchinskii and Lounici (2014).

######  Lemma 4.

$\displaystyle\mathbb{E}\|\widehat{M}-M\|\asymp\|M\|\left(\sqrt{\frac{r(M)}{n}}\bigvee\frac{r(M)}{n}\right),$

and

$\displaystyle\mathbb{E}\|\widehat{M}-M\|^{2}\asymp\|M\|^{2}\left(\sqrt{\frac{r(M)}{n}}\bigvee\frac{r(M)}{n}\right)^{2}.$

Let $\widehat{\mathbf{u}}_{1}$ be the leading eigenvector of $\widehat{M}$.
Without loss of generality, to make the linear form of
$\widehat{\mathbf{u}}_{1}$ well-defined, we always assume that
$\langle\widehat{\mathbf{u}}_{1},\mathbf{u}_{1}\rangle\geq 0$. Denote
$\widehat{{\cal
P}}_{1}:=\widehat{\mathbf{u}}_{1}\otimes\widehat{\mathbf{u}}_{1}$. Define
$\bar{g}_{1}:=\mu_{1}-\mu_{2}$, the spectral gap of $\mu_{1}$, and write

$\displaystyle{\cal C}_{1}=\sum_{s\neq 1}\frac{1}{\mu_{1}-\mu_{s}}{\cal
P}_{s},$ (68) $\displaystyle{\cal L}_{1}$ $\displaystyle:={\cal
C}_{1}(\widehat{M}-M){\cal P}_{1}+{\cal P}_{1}(\widehat{M}-M){\cal
C}_{1}=\frac{1}{n}\sum_{j=1}^{n}({\cal C}_{1}Y_{j}\otimes{\cal
P}_{1}Y_{j}+{\cal P}_{1}Y_{j}\otimes{\cal C}_{1}Y_{j}),$ (69)
$\displaystyle{\cal S}_{1}$ $\displaystyle:=\widehat{{\cal P}}_{1}-{\cal
P}_{1}-{\cal L}_{1},$ (70)

and the remainder in terms of operator

$\displaystyle{\cal R}_{1}:=\widehat{{\cal P}}_{1}-\mathbb{E}\widehat{{\cal
P}}_{1}-{\cal L}_{1}.$ (71)

Note that $\mathbb{E}{\cal L}_{1}=0$, so ${\cal R}_{1}={\cal
S}_{1}-\mathbb{E}{\cal S}_{1}$. As in the proof of Theorem 4.4, we use
calligraphic capital letters on $\widehat{{\cal P}}_{1}$, ${\cal P}_{1}$,
${\cal C}_{1}$, ${\cal L}_{1}$, ${\cal S}_{1}$ and ${\cal R}_{1}$ to signify
that they are matrices. Rephrasing Lemma 1 in Koltchinskii and Lounici (2014),
we have

######  Lemma 5.

$\displaystyle\|{\cal S}_{1}\|\leq
14\left(\frac{\|\widehat{M}-M\|}{\bar{g}_{1}}\right)^{2}$ (72)

Combine Lemma 5 and 4, we have

$\displaystyle\mathbb{E}\|{\cal S}_{1}\|\leq
C\frac{\|M\|^{2}}{\bar{g}_{1}^{2}}\left(\sqrt{\frac{r(M)}{n}}\bigvee\frac{r(M)}{n}\right)^{2},$
(73)

where $C$ is a universal constant. Restating Theorems 3 and 4 of Koltchinskii
and Lounici (2014), we get

######  Lemma 6.

Let $t>1$ and suppose that, for some $\gamma\in(0,1)$ and a sufficiently large
constant $C>0$,

$\displaystyle\mathbb{E}\|\widehat{M}-M\|+C\|M\|\sqrt{\frac{t}{n}}\leq\frac{1-\gamma}{1+\gamma}\frac{\bar{g}_{1}}{2}.$
(74)

Then there exists a constant $D_{\gamma}>0$ such that, for all
$u,v\in\mathbb{H}$, the following bound holds with probability at least
$1-e^{-t}$ :

$\displaystyle\left|\big{\langle}(\widehat{{\cal
P}}_{1}-\mathbb{E}\widehat{{\cal P}}_{1}-{\cal
L}_{1})u,v\big{\rangle}\right|\leq
D_{\gamma}\frac{\|M\|^{2}}{\bar{g}_{1}^{2}}\left(\sqrt{\frac{r(M)}{n}}\bigvee\sqrt{\frac{t}{n}}\right)\sqrt{\frac{t}{n}}\|u\|\|v\|.$
(75)

######  Lemma 7.

Suppose that for some $\gamma\in(0,1)$ and a sufficiently large constant
$C>0$,

$\displaystyle\mathbb{E}\|\widehat{M}-M\|+C\|M\|\frac{\log
n}{n}\leq(1-\gamma)\frac{\bar{g}_{1}}{2}.$ (76)

Then, there exists a constant $D_{\gamma}>0$ such that

$\displaystyle\|\mathbb{E}\widehat{{\cal P}}_{1}-{\cal P}_{1}-{\cal
P}_{1}\mathbb{E}({\cal S}_{1}){\cal P}_{1}\|\leq
D_{\gamma}\frac{\|M\|^{2}}{\bar{g}_{1}^{2}}\frac{1}{\sqrt{n}}\left(\sqrt{\frac{r(M)}{n}}\bigvee\sqrt{\frac{\log
n}{n}}\right).$ (77)

With ${\cal S}_{1}$ defined, we can define a critical quantity that
characterizes the bias of $\widehat{{\cal P}}_{1}$:

$\displaystyle b_{1}:=\left\langle\mathbb{E}({\cal
S}_{1})\mathbf{u}_{1},\mathbf{u}_{1}\right\rangle.$ (78)

Note that $\mathbb{E}{\cal L}_{1}=0$ and $\widehat{{\cal
P}}_{1}:=\widehat{\mathbf{u}}_{1}\otimes\widehat{\mathbf{u}}_{1}$, in which
$\langle\widehat{\mathbf{u}}_{1},\mathbf{u}_{1}\rangle\geq 0$, we have

$\displaystyle
b_{1}=\left\langle\mathbb{E}(\widehat{\mathbf{u}}_{1}\otimes\widehat{\mathbf{u}}_{1})\mathbf{u}_{1},\mathbf{u}_{1}\right\rangle-1,$

so $-1\leq b_{1}\leq 0$.

There is a way to estimate $b_{1}$. Suppose we divide the sample
$(Y_{1},Y_{2},\dots,Y_{n})$ into two subsamples of sample size
$\lfloor\frac{n}{2}\rfloor$ each. Let $\check{M}$ be the sample covariance
based on the first subsample and $\check{M}^{\prime}$ be the sample covariance
based on the second subsample. Denote by $\check{\mathbf{u}}_{1}$ the leading
eigenvector of $\check{M}$ and by $\check{\mathbf{u}}_{1}^{\prime}$ the
leading eigenvector of $\check{M}^{\prime}$. Assume that their signs are
chosen in such a way that
$\langle\check{\mathbf{u}}_{1},\check{\mathbf{u}}_{1}^{\prime}\rangle\geq 0$.
Define

$\displaystyle\widehat{b}_{1}:=\langle\check{\mathbf{u}}_{1},\check{\mathbf{u}}_{1}^{\prime}\rangle-1.$
(79)

The following lemma, a restatement of Proposition 3 from Koltchinskii and
Lounici (2014), provides a concentration inequality of
$|\widehat{b}_{1}-b_{1}|$.

######  Lemma 8.

Let $t\geq 1$ and $\gamma\in(0,1/2)$. There exists a constant $D_{\gamma}>0$
such that, if

$\displaystyle\mathbb{E}\|\widehat{M}-M\|\leq\frac{(1-2\gamma)\bar{g}_{1}}{2},\
1+b_{1}\geq 2\gamma$ (80)

and

$\displaystyle D_{\gamma}\|M\|\left(\sqrt{\frac{t}{n}}\bigvee\sqrt{\frac{\log
n}{n}}\right)\leq\frac{\gamma\bar{g}_{1}}{2},$ (81)

then with probability at least $1-e^{-t}$,

$\displaystyle|\widehat{b}_{1}-b_{1}|\leq
D_{\gamma}\frac{\|M\|^{2}}{\bar{g}_{1}^{2}}\left(\sqrt{\frac{r(M)}{n}}\bigvee\sqrt{\frac{t}{n}}\bigvee\sqrt{\frac{\log
n}{n}}\right)\sqrt{\frac{t}{n}}.$ (82)

##### Proof outline.

Without loss of generality, we assume $\|\mathbf{u}\|,\|\mathbf{v}\|=1$
throughout the proof. Write

$\displaystyle{\cal L}_{j}:=\frac{2}{n}\sum_{k=1}^{n/2}\left({\cal
C}_{j}y_{jk}\otimes{\cal P}_{j}y_{jk}+{\cal P}_{j}y_{jk}\otimes{\cal
C}_{j}y_{jk}\right),$ (83)

then

$\displaystyle\langle{\cal
L}_{j}\mathbf{u},\mathbf{v}\rangle=\frac{2}{n}\sum_{k=1}^{n/2}\left[\langle
y_{jk},{\cal P}_{j}\mathbf{v}\rangle\langle y_{jk},{\cal
C}_{j}\mathbf{u}\rangle+\langle y_{jk},{\cal P}_{j}\mathbf{u}\rangle\langle
y_{jk},{\cal C}_{j}\mathbf{v}\rangle\right].$ (84)

Now Lemma 3 is equivalent to: there exists universal constant $C$ such that

$\displaystyle\mathbb{P}(|b_{j}^{(q)[1]}|\leq C\frac{d}{n})\to 1,$ (85)

and for $\forall\mathbf{u},\mathbf{v}\in\mathbb{H}$,
$\|\mathbf{u}\|,\|\mathbf{v}\|\leq 1$,

$\displaystyle\sqrt{\frac{n}{2}}\bigg{\langle}\left[\widehat{{\cal
P}}_{j}-{\cal P}_{j}-{\cal P}_{j}\bar{\mathbb{E}}({\cal S}^{[1]}){\cal
P}_{j}-{\cal
L}_{j}\right]\mathbf{u},\mathbf{v}\bigg{\rangle}\overset{p}{\to}0.$ (86)

We separate the proof for (86) into three parts:

$\displaystyle\sqrt{\frac{n}{2}}\big{\langle}(\widehat{{\cal
P}}_{j}-\bar{\mathbb{E}}\widehat{{\cal P}}_{j}-{\cal
L}^{[1]})u,v\big{\rangle}\overset{p}{\to}0,$ (87)
$\displaystyle\sqrt{\frac{n}{2}}\left\|\bar{\mathbb{E}}\widehat{{\cal
P}}_{j}-{\cal P}_{j}-{\cal P}_{j}\bar{\mathbb{E}}(S^{[1]}){\cal
P}_{j}\right\|\overset{p}{\to}0,$ (88)
$\displaystyle\sqrt{\frac{n}{2}}\big{\langle}({\cal L}^{[1]}-{\cal
L}_{j})u,v\big{\rangle}\overset{p}{\to}0.$ (89)

We will first prove some preliminary bounds and (85), and then come back to
(87), (88) and (89) to complete the proof.

###  E.2 Proof for (85).

Upper bounds (16) and (17) imply that $\widehat{\mathbf{u}}_{j}^{(q)[2]}$
satisfies the following conditions:

$\displaystyle\lim_{n\to\infty}\mathbb{P}\left(\|\widehat{\mathbf{u}}_{j}^{(q)[2]}-\mathbf{u}_{j}^{(q)}\|\leq
a_{n},\ \forall q\in[p]\right)=1,$ (90)

where $a_{n}$ is a numeric sequence such that $\lim_{n\to\infty}a_{n}=0$, and

$\displaystyle\lim_{n\to\infty}\mathbb{P}\left(\max_{l\neq
j}\left|\sigma_{l}^{2}\langle\mathbf{u}_{l}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right|\leq
A,\ \forall q\in[p]\right)=1,$ (91)

where $A$ is a numeric constant. Define events

$\mathcal{B}_{n}:=\left\\{\|\widehat{\mathbf{u}}_{j}^{(q)[2]}-\mathbf{u}_{j}^{(q)}\|\leq
a_{n},\ \|\widehat{\mathbf{u}}_{j}^{(q)[2]}-\mathbf{u}_{j}^{(q)}\|\leq a_{n},\
\forall q\in[p]\right\\}.$

By (90) and (91), $\lim_{n\to\infty}\mathbb{P}(\mathcal{B}_{n})=1$. Note that
event $\mathcal{B}_{n}$ belongs to the sigma field of
$\\{\mathscr{X}_{n/2+1},\mathscr{X}_{n/2+2},\dots,\mathscr{X}_{n}\\}$, so it
suffices to treat $\bar{\mathbb{P}}(\cdot)$ and $\bar{\mathbb{E}}(\cdot)$ as
conditional on the event $\mathcal{B}_{n}$. Since events $\mathcal{B}_{n}$
satisfy $\lim_{n\to\infty}\mathbb{P}(\mathcal{B}_{n})=1$, for any sequence of
random variables $Z_{n}$, to prove $Z_{n}\overset{p}{\to}0$, we only need to
show $Z_{n}\cdot\mathbb{I}(\mathcal{B}_{n})\overset{p}{\to}0$, where
$\mathbb{I}(\mathcal{B}_{n})$ is the indicator function of event
$\mathcal{B}_{n}$. We will use this technique extensively.

We only need to prove for the case $q=1$. From now on till the end of this
proof, for simplicity of notations, we denote ${\cal P}_{j}={\cal
P}_{j}^{(1)}$, ${\cal C}_{j}={\cal C}_{j}^{(1)}$, $\widehat{{\cal
P}}_{j}=\widehat{{\cal P}}_{j}^{(1)}$, $M_{j}^{[1]}=M^{(1)[1]}$, ${\cal
C}_{j}^{[1]}={\cal C}_{j}^{(1)[1]}$, ${\cal L}_{j}^{[1]}={\cal
L}_{j}^{(1)[1]}$, ${\cal S}_{j}^{[1]}={\cal S}_{j}^{(1)[1]}$. Furthermore,
write

$\displaystyle{\cal
P}_{-}=I_{d_{q}}-\sum_{l=1}^{r}\mathbf{u}_{l}^{(q)}\otimes\mathbf{u}_{l}^{(q)}$
(92)

Note that ${\cal C}_{j}^{[1]},{\cal L}_{j}^{[1]},{\cal S}_{j}^{[1]},R^{[1]}$
are the ${\cal C}_{1},{\cal L}_{1},{\cal S}_{1},{\cal R}_{1}$ [defined in
(68), (69), (70), (71)] corresponding to our covariance matrix $M_{j}^{[1]}$.
Conditional on $\\{\mathscr{X}_{k},k=n/2+1,\dots,n\\}$,

$\displaystyle
z_{jk}:=\mathscr{X}_{k}\times_{2}\widehat{\mathbf{u}}_{j}^{(1)[2]}\dots\times_{p}\widehat{\mathbf{u}}_{j}^{(p)[2]},k=1,\dots,n/2$

has covariance matrix $M_{j}^{[1]}$, and $\widehat{\mathbf{u}}_{j}^{(q)[1]}$
is the leading eigenvector of the sample covariance matrix

$\displaystyle\widehat{M}_{j}^{[1]}=\frac{2}{n}\sum_{k=1}^{n/2}z_{jk}\otimes
z_{jk}.$ (93)

We first prove some inequalities for $\|M_{j}^{[1]}\|$, $\bar{g}_{1}$ the
first spectral gap of $M_{j}^{[1]}$, and $r(M_{j}^{[1]})$ the effective rank
of $M_{j}^{[1]}$, under event $\mathcal{B}_{n}$. These inequalities will be
used extensively throughout the proof.

Under $\mathcal{B}_{n}$, since
$\|\widehat{\mathbf{u}}_{j}^{(q)[2]}-\mathbf{u}_{j}^{(q)}\|\leq a_{n}$, we
have
$(1-a_{n}^{2})^{\frac{p}{2}}\sigma_{j}^{2}\leq\tilde{\sigma}_{j}^{2}\leq\sigma_{j}^{2}$.
Moreover, because $\max_{l\neq
j}\left|\sigma_{l}^{2}\langle\mathbf{u}_{l}^{(2)},\widehat{\mathbf{u}}_{j}^{(2)[2]}\rangle\right|\leq
A$ is bounded, and
$\left|\langle\mathbf{u}_{l}^{(2)},\widehat{\mathbf{u}}_{j}^{(2)[2]}\rangle\right|\leq\sqrt{1-\langle\widehat{\mathbf{u}}_{j}^{(2)[2]},\mathbf{u}_{j}^{(2)}\rangle^{2}}\leq
a_{n}$, we have $\max_{l\neq j}\\{\tilde{\sigma}_{l}^{2}\\}\leq Aa_{n}$. Since
$\lim_{n\to\infty}a_{n}=0$, for large enough $n$, we have
$(1-a_{n}^{2})^{\frac{p}{2}}\sigma_{j}^{2}>Aa_{n}$. So the leading eigenvector
of $M_{j}^{[1]}$ is $\mathbf{u}_{j}^{(1)}$, with corresponding eigenvalue
$\tilde{\sigma}_{j}^{2}+1$. So for large enough $n$,

$\displaystyle\frac{1}{2}\sigma_{j}^{2}+1\leq(1-a_{n}^{2})^{\frac{p}{2}}\sigma_{j}^{2}+1\leq\|M_{j}^{[1]}\|\leq\sigma_{j}^{2}+1,$
(94)

and the spectral gap for the leading eigenvalue of $M_{j}^{[1]}$

$\displaystyle\bar{g}_{1}\geq(1-a_{n}^{2})^{\frac{p}{2}}\sigma_{j}^{2}-Aa_{n}.$
(95)

Since $\lim_{n\to\infty}a_{n}=0$, we have for large enough $n$,

$\displaystyle\bar{g}_{1}\geq\frac{1}{2}\sigma_{j}^{2}.$ (96)

By definition,

$\displaystyle r(M_{j}^{[1]})=\frac{{\rm tr}(M_{j}^{[1]})}{\|M_{j}^{[1]}\|}.$
(97)

Combine with (94), and

$\displaystyle{\rm tr}(M_{j}^{[1]})$ $\displaystyle=$
$\displaystyle\sum_{l\neq
j}\sigma_{l}^{2}\left(\prod_{q=2}^{p}\langle\mathbf{u}_{l}^{(q)},\widehat{\mathbf{u}}_{j}^{(q)[2]}\rangle\right)^{2}+\tilde{\sigma}_{j}^{2}+d_{1}$
$\displaystyle\leq$ $\displaystyle\max_{l\neq
j}\left|\sigma_{l}^{2}\langle\mathbf{u}_{l}^{(2)},\widehat{\mathbf{u}}_{j}^{(2)[2]}\rangle\right|+\sigma_{j}^{2}+d_{1}$
$\displaystyle\leq$ $\displaystyle Aa_{n}+\sigma_{j}^{2}+d_{1}$

where the first inequality is by Cauchy inequality. So for large enough $n$,

$\displaystyle\frac{d_{1}}{\sigma_{j}^{2}+1}\leq
r(M_{j}^{[1]})\leq\frac{\sigma_{j}^{2}+d_{1}+1}{\frac{1}{2}\sigma_{j}^{2}+1}.$
(98)

Combine inequalities (73) from Lemma 6 with bounds (94), (96) and (98), we
have that under event $\mathcal{B}_{n}$, for large enough $n$,

$\displaystyle\left|b_{j}^{(1)[1]}\right|\leq\left\|\bar{\mathbb{E}}({\cal
S}_{j}^{[1]})\right\|\leq\bar{\mathbb{E}}\left\|{\cal S}_{j}^{[1]}\right\|\leq
C\frac{(\sigma_{j}^{2}+1)^{2}}{\frac{1}{4}\sigma_{j}^{4}}\left(\sqrt{\frac{\sigma_{j}^{2}+d_{1}+1}{n(\frac{1}{2}\sigma_{j}^{2}+1)}}\bigvee\frac{\sigma_{j}^{2}+d_{1}+1}{n(\frac{1}{2}\sigma_{j}^{2}+1)}\right)^{2},$
(99)

since we treat $\sigma_{j}$ as fixed, the right hand side is bounded by
$C\left(\sqrt{\frac{d}{n}}\bigvee\frac{d}{n}\right)^{2}\leq C\frac{d}{n}$.
Recall that $\lim_{n\to\infty}\mathbb{P}(\mathcal{B}_{n})=1$, we have
$\mathbb{P}(|b_{j}^{(1)[1]}|\leq C\frac{d}{n})\to 1$ as $n\to\infty$.

###  E.3 Proof for (87).

In this part, we will use Lemma 6 to show that

$\displaystyle\sqrt{\frac{n}{2}}\big{\langle}(\widehat{{\cal
P}}_{j}-\bar{\mathbb{E}}\widehat{{\cal P}}_{j}-{\cal
L}_{j}^{[1]})\mathbf{u},\mathbf{v}\big{\rangle}\overset{p}{\to}0.$

Let $t=\left(\frac{n}{2r(M_{j}^{[1]})}\right)^{\frac{1}{3}}$. First note that
on the event $\mathcal{B}_{n}$, by inequality (98),

$\displaystyle
t\geq\left(\frac{n(\frac{1}{2}\sigma_{j}^{2}+1)}{2(\sigma_{j}^{2}+d_{1}+1)}\right)^{\frac{1}{3}}\geq
1,$

for large enough $n$. By Lemma 4,

$\displaystyle\delta_{n}(t)$
$\displaystyle=\bar{\mathbb{E}}\|\widehat{M}_{j}^{[1]}-M_{j}^{[1]}\|+C\|M_{j}^{[1]}\|\sqrt{\frac{t}{n}}$
$\displaystyle\leq
C\|M_{j}^{[1]}\|\left(\sqrt{\frac{r(M_{j}^{[1]})}{n}}\bigvee\frac{r(M_{j}^{[1]})}{n}\right)+C\|M_{j}^{[1]}\|\sqrt{\frac{t}{n}}$
$\displaystyle\leq
C\|M_{j}^{[1]}\|\left[\frac{4(\sigma_{j}^{2}+d_{1}+1)}{(\frac{1}{2}\sigma_{j}^{2}+1)n}+\sqrt{\frac{2\left(\frac{n}{2r(M_{j}^{[1]})}\right)^{\frac{1}{3}}}{n}}\right]$
$\displaystyle\leq
C(\sigma_{j}^{2}+1)\left[\frac{\sigma_{j}^{2}+d_{1}+1}{(\frac{1}{2}\sigma_{j}^{2}+1)n}+n^{-\frac{1}{3}}\right],$

on events $\mathcal{B}_{n}$. The last inequality holds because of (98) and
(94). Recall that $\bar{g}_{1}\geq\frac{1}{2}\sigma_{j}^{2}$. We have that for
large enough $n$, $\delta_{n}(t)\leq\frac{1}{6}\bar{g}_{1}$.

Now, by Lemma 6 with $t=\left(\frac{n}{2r(M_{j}^{[1]})}\right)^{\frac{1}{3}}$
and $\gamma=\frac{1}{2}$, there exists a constant $D:=D_{\frac{1}{2}}$ such
that on the event $\mathcal{B}_{n}$, conditional on
$\\{\mathscr{X}_{n/2+1},\mathscr{X}_{n/2+2},\dots,\mathscr{X}_{n}\\}$, with
probability at least $1-e^{-t}\geq
1-\exp\left[-\left(\frac{n(\frac{1}{2}\sigma_{j}^{2}+1)}{2(\sigma_{j}^{2}+d_{1}+1)}\right)^{\frac{1}{3}}\right]$
:

$\displaystyle\sqrt{\frac{n}{2}}\left|\big{\langle}(\widehat{{\cal
P}}_{j}-\bar{\mathbb{E}}\widehat{{\cal P}}_{j}-{\cal
L}_{j}^{[1]})\mathbf{u},\mathbf{v}\big{\rangle}\right|$ $\displaystyle\leq$
$\displaystyle
D\sqrt{\frac{n}{2}}\frac{\|M_{j}^{[1]}\|^{2}}{\bar{g}_{1}^{2}}\left(\sqrt{\frac{r(M_{j}^{[1]})}{n}}\bigvee\sqrt{\frac{t}{n}}\right)\sqrt{\frac{t}{n}}$
$\displaystyle=$ $\displaystyle
D\frac{\|M_{j}^{[1]}\|^{2}}{\bar{g}_{1}^{2}}\left[\left(\frac{r(M_{j}^{[1]})}{n}\right)^{\frac{1}{3}}\bigvee\frac{1}{r(M_{j}^{[1]})^{\frac{1}{3}}(n/2)^{\frac{1}{6}}}\
\right]$ $\displaystyle\leq$ $\displaystyle
4D\left(\frac{\sigma_{j}^{2}+1}{\frac{1}{2}\sigma_{j}^{2}}\right)^{2}\left[\left(\frac{\sigma_{j}^{2}+d_{1}+1}{n(\frac{1}{2}\sigma_{j}^{2}+1)}\right)^{\frac{1}{3}}\bigvee\frac{1}{n^{\frac{1}{6}}}\
\right],$ (100)

conditional on $\\{\mathscr{X}_{k},k=n/2+1,\dots,n\\}$ under events
$\mathcal{B}_{n}$, for large enough $n$, with probability at least $1-e^{-t}$.
The last inequality holds because of (94), (96) and (98).

Since on the event $\mathcal{B}_{n}$, conditional on
$\\{\mathscr{X}_{n/2+1},\mathscr{X}_{n/2+2},\dots,\mathscr{X}_{n}\\}$, (100)
holds with probability at least
$1-\exp\left[-\left(\frac{n(\frac{1}{2}\sigma_{j}^{2}+1)}{2(\sigma_{j}^{2}+d_{1}+1)}\right)^{\frac{1}{3}}\right]$,
we have that on the event $\mathcal{B}_{n}$, (100) also holds with probability
at least
$1-\exp\left[-\left(\frac{n(\frac{1}{2}\sigma_{j}^{2}+1)}{2(\sigma_{j}^{2}+d_{1}+1)}\right)^{\frac{1}{3}}\right]$.
Combining with the assumption that ${d}/{n}\to 0$, we have

$\displaystyle\sqrt{\frac{n}{2}}\left|\big{\langle}(\widehat{{\cal
P}}_{j}-\bar{\mathbb{E}}\widehat{{\cal P}}_{j}-{\cal
L}_{j}^{[1]})\mathbf{u},\mathbf{v}\big{\rangle}\right|\cdot\mathbb{I}(\mathcal{B}_{n})\overset{p}{\to}0.$

###  E.4 Proof for (88).

In this part, we will use Lemma 7 to show that

$\displaystyle\sqrt{\frac{n}{2}}\left\|\bar{\mathbb{E}}\widehat{{\cal
P}}_{j}-{\cal P}_{j}-{\cal P}_{j}\bar{\mathbb{E}}({\cal S}_{j}^{[1]}){\cal
P}_{j}\right\|\overset{p}{\to}0.$

By Lemma 4,

$\displaystyle\bar{\mathbb{E}}\|\widehat{M}_{j}^{[1]}-M_{j}^{[1]}\|+C\|M_{j}^{[1]}\|\frac{\log
n}{n}$ $\displaystyle\leq$ $\displaystyle
C\|M_{j}^{[1]}\|\left(\sqrt{\frac{r(M_{j}^{[1]})}{n}}\bigvee\frac{r(M_{j}^{[1]})}{n}\right)+C\|M_{j}^{[1]}\|\frac{\log
n}{n}$ $\displaystyle\leq$ $\displaystyle
C(\sigma_{j}^{2}+1)\left[\frac{\sigma_{j}^{2}+d_{1}+1}{(\frac{1}{2}\sigma_{j}^{2}+1)n}+\frac{\log
n}{n}\right],$

on events $\mathcal{B}_{n}$. The last inequality holds because of (98) and
(94). Remember that $\bar{g}_{1}\geq\frac{1}{2}\sigma_{j}^{2}$ (inequality
96), since $d=o(n)$, we have that for large enough $n$,
$\delta_{n}(t)\leq\frac{1}{4}\bar{g}_{1}$.

Now, by Lemma 7, with $\gamma=\frac{1}{2}$, there exists a constant
$D:=D_{\frac{1}{2}}$ such that on the event $\mathcal{B}_{n}$:

$\displaystyle\sqrt{\frac{n}{2}}\left\|\bar{\mathbb{E}}\widehat{{\cal
P}}_{j}-{\cal P}_{j}-{\cal P}_{j}\bar{\mathbb{E}}({\cal S}_{j}^{[1]}){\cal
P}_{j}\right\|$ $\displaystyle=$ $\displaystyle
D\sqrt{n/2}\frac{\|M_{j}^{[1]}\|^{2}}{\bar{g}_{1}^{2}}\frac{1}{\sqrt{n/2}}\left(\sqrt{\frac{r(M_{j}^{[1]})}{n}}\bigvee\sqrt{\frac{\log
n}{n}}\right)$ $\displaystyle\leq$ $\displaystyle
4D\left(\frac{\sigma_{j}^{2}+1}{\frac{1}{2}\sigma_{j}^{2}}\right)^{2}\left[\sqrt{\frac{\sigma_{j}^{2}+d_{1}+1}{n(\frac{1}{2}\sigma_{j}^{2}+1)}}\bigvee\sqrt{\frac{\log
n}{n}}\right],$ (101)

for large enough $n$. The last inequality holds because of (94), (96) and
(98). Equation (88) then follows.

###  E.5 Proof for (89).

Note that for $\mathbf{u},\mathbf{v}\in\mathbb{H}$,

$\displaystyle\langle{\cal
L}_{j}^{[1]}\mathbf{u},\mathbf{v}\rangle=\frac{2}{n}\sum_{k=1}^{n/2}\left[\langle
z_{jk},{\cal P}_{j}\mathbf{v}\rangle\langle z_{jk},{\cal
C}_{j}^{[1]}\mathbf{u}\rangle+\langle z_{jk},{\cal
P}_{j}\mathbf{u}\rangle\langle z_{jk},{\cal
C}_{j}^{[1]}\mathbf{v}\rangle\right],$ (102)

and remember that $\langle
L_{j}\mathbf{u},\mathbf{v}\rangle=\frac{2}{n}\sum_{k=1}^{n/2}\left[\langle
y_{jk},{\cal P}_{j}\mathbf{v}\rangle\langle y_{jk},{\cal
C}_{j}\mathbf{u}\rangle+\langle y_{jk},{\cal P}_{j}\mathbf{u}\rangle\langle
y_{jk},{\cal C}_{j}\mathbf{v}\rangle\right]$, so to prove (89), we only need
to show: for $\forall\mathbf{u},\mathbf{v}\in\mathbb{H}$,
$\|\mathbf{u}\|,\|\mathbf{v}\|\leq 1$,

$\displaystyle\sqrt{\frac{2}{n}}\sum_{k=1}^{n/2}\left[\langle z_{jk},{\cal
P}_{j}\mathbf{v}\rangle\langle z_{jk},{\cal
C}_{j}^{[1]}\mathbf{u}\rangle-\langle y_{jk},{\cal
P}_{j}\mathbf{v}\rangle\langle y_{jk},{\cal
C}_{j}\mathbf{u}\rangle\right]\overset{p}{\to}0.$ (103)

We further break the proof for (103) into three steps.

##### Step 1.

$\displaystyle\langle z_{jk},{\cal P}_{j}\mathbf{v}\rangle\langle z_{jk},{\cal
C}_{j}^{[1]}\mathbf{u}\rangle-\langle y_{jk},{\cal
P}_{j}\mathbf{v}\rangle\langle z_{jk},{\cal C}_{j}^{[1]}\mathbf{u}\rangle$
$\displaystyle=$ $\displaystyle\langle z_{jk}-y_{jk},{\cal
P}_{j}\mathbf{v}\rangle\langle z_{jk},{\cal C}_{j}^{[1]}\mathbf{u}\rangle$
(104)

but conditional on
$\\{\mathscr{X}_{n/2+1},\mathscr{X}_{n/2+2},\dots,\mathscr{X}_{n}\\}$,
$\langle z_{jk}-y_{jk},{\cal P}_{j}\mathbf{v}\rangle$ and $\langle
z_{jk},{\cal C}_{j}^{[1]}\mathbf{u}\rangle$, $k=1,\dots,n$ are mean-$0$
Gaussian random variables and uncorrelated hence independent. So
$\bar{\mathbb{E}}(\langle z_{jk}-y_{jk},{\cal P}_{j}\mathbf{v}\rangle\langle
z_{jk},{\cal C}_{j}^{[1]}\mathbf{u}\rangle)=0$. Moreover, direct calculation
gives us

$\displaystyle\bar{\mathbb{E}}\langle z_{jk}-y_{jk},{\cal
P}_{j}\mathbf{v}\rangle^{2}$ $\displaystyle=$
$\displaystyle\langle\mathbf{u}_{j}^{(1)},\mathbf{v}\rangle^{2}\bar{\mathbb{E}}\bigg{[}\mathscr{X}_{k}\times_{1}\mathbf{u}_{j}^{(1)}\times_{2}\widehat{\mathbf{u}}_{j}^{(2)[2]}\dots\times_{p}\widehat{\mathbf{u}}_{j}^{(p)[2]}-\mathscr{X}_{k}\times_{1}\mathbf{u}_{j}^{(1)}\times_{2}\mathbf{u}_{j}^{(2)}\dots\times_{p}\mathbf{u}_{j}^{(p)}\bigg{]}^{2}$
$\displaystyle=$
$\displaystyle\langle\mathbf{u}_{j}^{(1)},\mathbf{v}\rangle^{2}\sigma_{j}^{2}\prod_{q=2}^{r}\left(1-\langle\widehat{\mathbf{u}}_{j}^{(q)[2]},\mathbf{u}_{j}^{(q)}\rangle\right)^{2}+\langle\mathbf{u}_{j}^{(1)},\mathbf{v}\rangle^{2}\left\|\widehat{\mathbf{u}}_{j}^{(2)[2]}\otimes\dots\otimes\widehat{\mathbf{u}}_{j}^{(p)[2]}-\mathbf{u}_{j}^{(2)}\otimes\dots\otimes\mathbf{u}_{j}^{(p)}\right\|^{2}$
$\displaystyle=$
$\displaystyle\langle\mathbf{u}_{j}^{(1)},\mathbf{v}\rangle^{2}\sigma_{j}^{2}\prod_{q=2}^{r}\left(1-\langle\widehat{\mathbf{u}}_{j}^{(q)[2]},\mathbf{u}_{j}^{(q)}\rangle\right)^{2}+\langle\mathbf{u}_{j}^{(1)},\mathbf{v}\rangle^{2}\left(2-2\prod_{q=2}^{r}\langle\widehat{\mathbf{u}}_{j}^{(q)[2]},\mathbf{u}_{j}^{(q)}\rangle\right)$
$\displaystyle\leq$
$\displaystyle\sigma_{j}^{2}\left(\frac{1}{4}a_{n}^{4}\right)^{p-1}+2-2\left(1-\frac{1}{2}a_{n}^{2}\right)^{p-1}$
(105)

since $\|\widehat{\mathbf{u}}_{j}^{(q)[2]}-\mathbf{u}_{j}^{(q)}\|\leq a_{n}$
which implies
$1-\langle\widehat{\mathbf{u}}_{j}^{(q)[2]},\mathbf{u}_{j}^{(q)}\rangle\leq\frac{1}{2}a_{n}^{2}$.
Also we have

$\displaystyle\bar{\mathbb{E}}\left[\langle z_{jk},{\cal
C}_{j}^{[1]}\mathbf{u}\rangle\right]^{2}=\langle{\cal
C}_{j}^{[1]}M_{j}^{[1]}{\cal C}_{j}^{[1]}\mathbf{u},\mathbf{u}\rangle,$ (106)

where

$\displaystyle{\cal C}_{j}^{[1]}M_{j}^{[1]}{\cal C}_{j}^{[1]}:=\sum_{l\neq
j}\frac{\tilde{\sigma}_{l}^{2}}{\left(\tilde{\sigma}_{j}^{2}-\tilde{\sigma}_{l}^{2}\right)^{2}}\mathbf{u}_{l}^{(1)}\otimes\mathbf{u}_{l}^{(1)}+\frac{1}{\tilde{\sigma}_{j}^{4}}{\cal
P}_{-}.$ (107)

As already proven,
$(1-a_{n}^{2})^{\frac{p}{2}}\sigma_{j}^{2}\leq\tilde{\sigma}_{j}^{2}$, and
$\max_{l\neq j}\\{\tilde{\sigma}_{l}^{2}\\}\leq Aa_{n}\to 0$, and for large
enough $n$,
$\tilde{\sigma}_{j}^{2}-\tilde{\sigma}_{l}^{2}\geq\frac{1}{2}\sigma_{j}^{2}$,
so for large enough $n$, we have

$\displaystyle\left\|{\cal C}_{j}^{[1]}M_{j}^{[1]}{\cal
C}_{j}^{[1]}\right\|\leq\frac{1}{\left[(1-a_{n}^{2})^{\frac{p}{2}}\sigma_{j}^{2}\right]^{4}}.$
(108)

Thus,

$\displaystyle\bar{\mathbb{E}}\left[\langle z_{jk}-y_{jk},{\cal
P}_{j}\mathbf{v}\rangle\langle z_{jk},{\cal
C}_{j}^{[1]}\mathbf{u}\rangle\right]^{2}$ $\displaystyle\leq$
$\displaystyle\frac{1}{\left[(1-a_{n}^{2})^{\frac{p}{2}}\sigma_{j}^{2}\right]^{4}}\left[\sigma_{j}^{2}\left(\frac{1}{4}a_{n}^{4}\right)^{p-1}+2-2\left(1-\frac{1}{2}a_{n}^{2}\right)^{p-1}\right].$
(109)

##### Step 2.

$\displaystyle\langle y_{jk},{\cal P}_{j}\mathbf{v}\rangle\langle z_{jk},{\cal
C}_{j}^{[1]}\mathbf{u}\rangle-\langle y_{jk},{\cal
P}_{j}\mathbf{v}\rangle\langle y_{jk},{\cal C}_{j}^{[1]}\mathbf{u}\rangle$
$\displaystyle=$ $\displaystyle\langle y_{jk},{\cal
P}_{j}\mathbf{v}\rangle\langle z_{jk}-y_{jk},{\cal
C}_{j}^{[1]}\mathbf{u}\rangle$ (110)

but conditional on
$\\{\mathscr{X}_{n/2+1},\mathscr{X}_{n/2+2},\dots,\mathscr{X}_{n}\\}$,
$\langle y_{jk},{\cal P}_{j}\mathbf{v}\rangle$ and $\langle
z_{jk}-y_{jk},{\cal C}_{j}^{[1]}\mathbf{u}\rangle$, $k=1,\dots,n$ are mean-$0$
Gaussian random variables and uncorrelated hence independent. So
$\bar{\mathbb{E}}(\langle z_{jk}-y_{jk},{\cal P}_{j}\mathbf{v}\rangle\langle
z_{jk},{\cal C}_{j}^{[1]}\mathbf{u}\rangle)=0$.

Moreover,

$\displaystyle\bar{\mathbb{E}}\langle z_{jk}-y_{jk},{\cal
C}_{j}^{[1]}\mathbf{u}\rangle^{2}$ $\displaystyle=$ $\displaystyle\sum_{l\neq
j}\sigma_{l}^{2}\langle{\cal
C}_{j}^{[1]}\mathbf{u},\mathbf{u}_{l}^{(1)}\rangle^{2}\prod_{q=2}^{r}\left(\langle\widehat{\mathbf{u}}_{j}^{(q)[2]},\mathbf{u}_{l}^{(q)}\rangle\right)^{2}$
$\displaystyle\qquad+\left\|{\cal
C}_{j}^{[1]}\mathbf{u}\right\|^{2}\cdot\left\|\widehat{\mathbf{u}}_{j}^{(2)[2]}\otimes\dots\otimes\widehat{\mathbf{u}}_{j}^{(p)[2]}-\mathbf{u}_{j}^{(2)}\otimes\dots\otimes\mathbf{u}_{j}^{(p)}\right\|^{2}$
$\displaystyle\leq$ $\displaystyle\left\|{\cal
C}_{j}^{[1]}\right\|^{2}\max_{l\neq
j}\left|\sigma_{l}^{2}\langle\mathbf{u}_{l}^{(2)},\widehat{\mathbf{u}}_{j}^{(2)[2]}\rangle\right|\sqrt{1-\langle\mathbf{u}_{j}^{(2)},\widehat{\mathbf{u}}_{j}^{(2)[2]}\rangle^{2}}+\left\|{\cal
C}_{j}^{[1]}\right\|^{2}\left(2-2\left(1-\frac{1}{2}a_{n}^{2}\right)^{p-1}\right)$
(111)

Since

$\displaystyle{\cal C}_{j}^{[1]}=\sum_{l\neq
j}\frac{1}{\tilde{\sigma}_{j}^{2}-\tilde{\sigma}_{l}^{2}}\mathbf{u}_{l}^{(1)}\otimes\mathbf{u}_{l}^{(1)}+\frac{1}{\tilde{\sigma}_{j}^{2}}{\cal
P}_{-},$ (112)

As already proven, for large enough $n$,
$\tilde{\sigma}_{j}^{2}-\tilde{\sigma}_{l}^{2}\geq\frac{1}{2}\sigma_{j}^{2}$,
so for large enough $n$, we have

$\displaystyle\left\|{\cal C}_{j}^{[1]}\right\|\leq\frac{2}{\sigma_{j}^{2}}.$
(113)

Because $\|\widehat{\mathbf{u}}_{j}^{(q)[2]}-\mathbf{u}_{j}^{(q)}\|\leq
a_{n}$, and $\max_{l\neq
j}\left|\sigma_{l}^{2}\langle\mathbf{u}_{l}^{(2)},\widehat{\mathbf{u}}_{j}^{(2)[2]}\rangle\right|\leq
A$,

$\displaystyle\bar{\mathbb{E}}\langle z_{jk}-y_{jk},{\cal
C}_{j}^{[1]}\mathbf{u}\rangle^{2}$ $\displaystyle\leq$
$\displaystyle\frac{2}{\sigma_{j}^{2}}\left[Aa_{n}+2-2\left(1-\frac{1}{2}a_{n}^{2}\right)^{p-1}\right]$
(114)

and $E\langle y_{jk},{\cal
P}_{j}\mathbf{v}\rangle^{2}=(\sigma_{j}^{2}+1)\langle\mathbf{u}_{j}^{(1)},v\rangle^{2}$,
so

$\displaystyle\bar{\mathbb{E}}\left[\langle y_{jk},{\cal
P}_{j}\mathbf{v}\rangle\langle z_{jk}-y_{jk},{\cal
C}_{j}^{[1]}\mathbf{u}\rangle\right]^{2}\leq(\sigma_{j}^{2}+1)\frac{2}{\sigma_{j}^{2}}\left[Aa_{n}+2-2\left(1-\frac{1}{2}a_{n}^{2}\right)^{p-1}\right]$
(115)

##### Step 3.

$\displaystyle\langle y_{jk},{\cal P}_{j}\mathbf{v}\rangle\langle y_{jk},{\cal
C}_{j}^{[1]}\mathbf{u}\rangle-\langle y_{jk},{\cal
P}_{j}\mathbf{v}\rangle\langle y_{jk},{\cal C}_{j}\mathbf{u}\rangle$
$\displaystyle=$ $\displaystyle\langle y_{jk},{\cal
P}_{j}\mathbf{v}\rangle\langle y_{jk},({\cal C}_{j}^{[1]}-{\cal
C}_{j})\mathbf{u}\rangle$ (116)

but conditional on
$\\{\mathscr{X}_{n/2+1},\mathscr{X}_{n/2+2},\dots,\mathscr{X}_{n}\\}$,
$\langle y_{jk},{\cal P}_{j}\mathbf{v}\rangle$ and $\langle y_{jk},({\cal
C}_{j}^{[1]}-{\cal C}_{j})\mathbf{u}\rangle$, $k=1,\dots,n$ are mean-$0$
Gaussian random variables and uncorrelated hence independent. So
$\bar{\mathbb{E}}(\langle y_{jk},{\cal P}_{j}\mathbf{v}\rangle\langle
y_{jk},({\cal C}_{j}^{[1]}-{\cal C}_{j})\mathbf{u}\rangle)=0$.

Moreover,

$\displaystyle{\cal C}_{j}^{[1]}-{\cal C}_{j}=\sum_{l\neq
j}\left(\frac{1}{\tilde{\sigma}_{j}^{2}-\tilde{\sigma}_{l}^{2}}-\frac{1}{\sigma_{j}^{2}}\right)\mathbf{u}_{l}^{(1)}\otimes\mathbf{u}_{l}^{(1)}+\left(\frac{1}{\tilde{\sigma}_{j}^{2}}-\frac{1}{\sigma_{j}^{2}}\right){\cal
P}_{-}-\frac{1}{\sigma_{j}^{2}}\left(I_{\mathbb{H}}-I_{d_{1}}\right)$ (117)

$\displaystyle\bar{\mathbb{E}}\langle y_{jk},({\cal C}_{j}^{[1]}-{\cal
C}_{j})\mathbf{u}\rangle^{2}$ $\displaystyle=$
$\displaystyle\left\|\left[\sum_{l\neq
j}\left(\frac{1}{\tilde{\sigma}_{j}^{2}-\tilde{\sigma}_{l}^{2}}-\frac{1}{\sigma_{j}^{2}}\right)\mathbf{u}_{l}^{(1)}\otimes\mathbf{u}_{l}^{(1)}+\left(\frac{1}{\tilde{\sigma}_{j}^{2}}-\frac{1}{\sigma_{j}^{2}}\right){\cal
P}_{-}\right]\mathbf{u}\right\|^{2}$ (118)

but as already proven,
$(1-a_{n}^{2})^{\frac{p}{2}}\sigma_{j}^{2}\leq\tilde{\sigma}_{j}^{2}$, and
$\max_{l\neq j}\\{\tilde{\sigma}_{l}^{2}\\}\leq Aa_{n}\to 0$, and for large
enough $n$,
$\tilde{\sigma}_{j}^{2}-\tilde{\sigma}_{l}^{2}\geq\frac{1}{2}\sigma_{j}^{2}$,
so for large enough $n$, we have

$\displaystyle\left\|\sum_{l\neq
j}\left(\frac{1}{\tilde{\sigma}_{j}^{2}-\tilde{\sigma}_{l}^{2}}-\frac{1}{\sigma_{j}^{2}}\right)\mathbf{u}_{l}^{(1)}\otimes\mathbf{u}_{l}^{(1)}+\left(\frac{1}{\tilde{\sigma}_{j}^{2}}-\frac{1}{\sigma_{j}^{2}}\right){\cal
P}_{-}\right\|$ $\displaystyle=$ $\displaystyle\max_{l\neq
j}\left(\frac{1}{\tilde{\sigma}_{j}^{2}-\tilde{\sigma}_{l}^{2}}-\frac{1}{\sigma_{j}^{2}}\right)\bigvee\left(\frac{1}{\tilde{\sigma}_{j}^{2}}-\frac{1}{\sigma_{j}^{2}}\right)$
$\displaystyle=$ $\displaystyle\max_{l\neq
j}\left(\frac{1}{\tilde{\sigma}_{j}^{2}-\tilde{\sigma}_{l}^{2}}-\frac{1}{\sigma_{j}^{2}}\right)$
$\displaystyle\leq$
$\displaystyle\frac{1}{(1-a_{n}^{2})^{\frac{p}{2}}\sigma_{j}^{2}-Aa_{n}}-\frac{1}{\sigma_{j}^{2}}$
(119)

and $E\langle y_{jk},{\cal
P}_{j}\mathbf{v}\rangle^{2}=(\sigma_{j}^{2}+1)\langle\mathbf{u}_{j}^{(1)},v\rangle^{2}$,
so

$\displaystyle\bar{\mathbb{E}}\left[\langle y_{jk},{\cal
P}_{j}\mathbf{v}\rangle\langle y_{jk},({\cal C}_{j}^{[1]}-{\cal
C}_{j})\mathbf{u}\rangle\right]^{2}\leq(\sigma_{j}^{2}+1)\left(\frac{1}{(1-a_{n}^{2})^{\frac{p}{2}}\sigma_{j}^{2}-Aa_{n}}-\frac{1}{\sigma_{j}^{2}}\right).$
(120)

Combining (109), (115), (120), we have that on events $\mathcal{B}_{n}$, for
large enough $n$,

$\displaystyle\bar{\mathbb{E}}\left(\langle z_{jk},{\cal
P}_{j}\mathbf{v}\rangle\langle z_{jk},{\cal
C}_{j}^{[1]}\mathbf{u}\rangle-\langle y_{jk},{\cal
P}_{j}\mathbf{v}\rangle\langle y_{jk},{\cal
C}_{j}\mathbf{u}\rangle\right)^{2}$ $\displaystyle\leq$
$\displaystyle\frac{1}{\left[(1-a_{n}^{2})^{\frac{p}{2}}\sigma_{j}^{2}\right]^{4}}\left[\sigma_{j}^{2}\left(\frac{1}{4}a_{n}^{4}\right)^{p-1}+2-2\left(1-\frac{1}{2}a_{n}^{2}\right)^{p-1}\right]$
$\displaystyle+(\sigma_{j}^{2}+1)\frac{2}{\sigma_{j}^{2}}\left[Aa_{n}+2-2\left(1-\frac{1}{2}a_{n}^{2}\right)^{p-1}\right]+(\sigma_{j}^{2}+1)\left(\frac{1}{(1-a_{n}^{2})^{\frac{p}{2}}\sigma_{j}^{2}-Aa_{n}}-\frac{1}{\sigma_{j}^{2}}\right).$

Note that conditional on
$\\{\mathscr{X}_{n/2+1},\mathscr{X}_{n/2+2},\dots,\mathscr{X}_{n}\\}$,

$\displaystyle\langle z_{jk},{\cal P}_{j}\mathbf{v}\rangle\langle z_{jk},{\cal
C}_{j}^{[1]}\mathbf{u}\rangle-\langle y_{jk},{\cal
P}_{j}\mathbf{v}\rangle\langle y_{jk},{\cal C}_{j}\mathbf{u}\rangle$

are independent with each other, so

$\displaystyle\bar{\mathbb{E}}\left(\sqrt{\frac{2}{n}}\sum_{k=1}^{n/2}\left[\langle
z_{jk},{\cal P}_{j}\mathbf{v}\rangle\langle z_{jk},{\cal
C}_{j}^{[1]}\mathbf{u}\rangle-\langle y_{jk},{\cal
P}_{j}\mathbf{v}\rangle\langle y_{jk},{\cal
C}_{j}\mathbf{u}\rangle\right]\right)^{2}$ $\displaystyle\leq$
$\displaystyle\frac{1}{\left[(1-a_{n}^{2})^{\frac{p}{2}}\sigma_{j}^{2}\right]^{4}}\left[\sigma_{j}^{2}\left(\frac{1}{4}a_{n}^{4}\right)^{p-1}+2-2\left(1-\frac{1}{2}a_{n}^{2}\right)^{p-1}\right]$
$\displaystyle+(\sigma_{j}^{2}+1)\frac{2}{\sigma_{j}^{2}}\left[Aa_{n}+2-2\left(1-\frac{1}{2}a_{n}^{2}\right)^{p-1}\right]+(\sigma_{j}^{2}+1)\left(\frac{1}{(1-a_{n}^{2})^{\frac{p}{2}}\sigma_{j}^{2}-Aa_{n}}-\frac{1}{\sigma_{j}^{2}}\right),$

on events $\mathcal{B}_{n}$, for large enough $n$. Combine with
$\lim_{n}a_{n}=0$, we have

$\displaystyle\mathbb{E}\left\\{\left(\sqrt{\frac{2}{n}}\sum_{k=1}^{n/2}\left[\langle
z_{jk},{\cal P}_{j}\mathbf{v}\rangle\langle z_{jk},{\cal
C}_{j}^{[1]}\mathbf{u}\rangle-\langle y_{jk},{\cal
P}_{j}\mathbf{v}\rangle\langle y_{jk},{\cal
C}_{j}\mathbf{u}\rangle\right]\right)\cdot\mathbb{I}(\mathcal{B}_{n})\right\\}^{2}$
$\displaystyle=$
$\displaystyle\mathbb{E}\left\\{\bar{\mathbb{E}}\left(\sqrt{\frac{2}{n}}\sum_{k=1}^{n/2}\left[\langle
z_{jk},{\cal P}_{j}\mathbf{v}\rangle\langle z_{jk},{\cal
C}_{j}^{[1]}\mathbf{u}\rangle-\langle y_{jk},{\cal
P}_{j}\mathbf{v}\rangle\langle y_{jk},{\cal
C}_{j}\mathbf{u}\rangle\right]\right)^{2}\cdot\mathbb{I}(\mathcal{B}_{n})\right\\}$
$\displaystyle\to$ $\displaystyle\ 0.$

So

$\displaystyle\left(\sqrt{\frac{2}{n}}\sum_{k=1}^{n/2}\left[\langle
z_{jk},{\cal P}_{j}\mathbf{v}\rangle\langle z_{jk},{\cal
C}_{j}^{[1]}\mathbf{u}\rangle-\langle y_{jk},{\cal
P}_{j}\mathbf{v}\rangle\langle y_{jk},{\cal
C}_{j}\mathbf{u}\rangle\right]\right)\cdot\mathbb{I}(\mathcal{B}_{n})\overset{p}{\to}0,$

thus proving (103), and (89) follows.