$\displaystyle(n^{-1}\bm{C}^{\top}P_{X}^{\perp}\bm{W}_{g}\bm{X})(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}(n^{-1}\bm{X}^{\top}\bm{W}_{g}P_{X}^{\perp}\bm{C})\\}\bm{R}^{-1/2},$

where
$\operatorname{\mathbb{E}}\\{\bm{R}^{-1/2}(n^{-1}\bm{C}^{\top}P_{X}^{\perp}\bm{W}_{g}P_{X}^{\perp}\bm{C})\bm{R}^{-1/2}\mid\bm{C}\\}=I_{K}$
and
$\operatorname{\mathbb{E}}(\bm{C}^{\top}P_{X}^{\perp}\bm{W}_{g}\bm{X}\mid\bm{C})=\bm{0}$.
First, Corollary S8.1 implies $\max_{g\in[p]}\lVert
n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X}-n^{-1}\bm{X}^{\top}\bm{X}\rVert_{2}=O_{P}(n^{-1/2+\epsilon})$.
Next,

$\displaystyle
n^{-1}\bm{C}^{\top}P_{X}^{\perp}\bm{W}_{g}\bm{X}=n^{-1}\bm{C}^{\top}\bm{W}_{g}\bm{X}-n^{-1}\bm{C}^{\top}\bm{X}(n^{-1}\bm{X}^{\top}\bm{X})^{-1}(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X}),$

where a second application of Corollary S8.1 shows that $\max_{g\in[p]}\lVert
n^{-1}\bm{C}^{\top}P_{X}^{\perp}\bm{W}_{g}\bm{X}\rVert_{2}=O_{P}(n^{-1/2+\epsilon})$.
Next,

$\displaystyle n^{-1}\bm{C}^{\top}P_{X}^{\perp}\bm{W}_{g}P_{X}^{\perp}\bm{C}=$
$\displaystyle n^{-1}\bm{C}^{\top}\bm{W}_{g}\bm{C}$
$\displaystyle+(n^{-1}\bm{C}^{\top}\bm{X})(n^{-1}\bm{X}^{\top}\bm{X})^{-1}(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X})(n^{-1}\bm{X}^{\top}\bm{X})^{-1}(n^{-1}\bm{X}^{\top}\bm{C})$
$\displaystyle-(n^{-1}\bm{C}^{\top}\bm{W}_{g}\bm{X})(n^{-1}\bm{X}^{\top}\bm{X})(n^{-1}\bm{X}^{\top}\bm{C})$
$\displaystyle-\\{(n^{-1}\bm{C}^{\top}\bm{W}_{g}\bm{X})(n^{-1}\bm{X}^{\top}\bm{X})(n^{-1}\bm{X}^{\top}\bm{C})\\}^{\top},$

where further applications of Corollary S8.1 to the terms in the above
expression imply

$\displaystyle\max_{g\in[p]}\lVert
n^{-1}\bm{C}^{\top}P_{X}^{\perp}\bm{W}_{g}P_{X}^{\perp}\bm{C}-\bm{R}\rVert_{2}=O_{P}(n^{-1/2+\epsilon}).$

This proves (S8.7). Since $\tilde{\bm{B}}_{g}=\bm{A}_{g}^{-1/2}\bm{B}_{g}$, it
suffices to only consider $\bm{B}_{g}^{\top}\bm{B}_{g}$ when proving (S8.8).
We have

$\displaystyle\bm{B}_{g}^{\top}\bm{B}_{g}=\bm{v}_{u}^{\top}\bm{A}_{g}\bm{v}_{u}+\bm{v}_{u}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{z}}_{u}+(\bm{v}_{u}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{z}}_{u})^{\top}+\tilde{\bm{z}}_{u}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\bm{A}_{g}^{-1}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{z}}_{u}.$

By Lemma S8.3 and (S8.7),
$\lVert\bm{v}_{u}^{\top}\bm{A}_{g}\bm{v}_{u}-I_{K}\rVert_{2}\leq
c\delta^{2}\\{1+O_{P}(n^{-1/2+\epsilon})\\}$ for some constant $c>0$. Since
$\lVert\bm{v}_{u}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{z}}_{u}\rVert_{2}\leq
c\delta\lVert\tilde{\bm{C}}^{\top}(\bm{P}_{g}^{\perp})^{2}\tilde{\bm{C}}\rVert_{2}^{1/2}$
for some constant $c>0$, we need only show that
$\lVert\tilde{\bm{C}}^{\top}(\bm{P}_{g}^{\perp})^{2}\tilde{\bm{C}}\rVert_{2}\leq
c\\{1+O_{P}(n^{-1/2+\epsilon})\\}$ for some constant $c>0$ to complete the
proof. However, this follows from an identical analysis used to study the
properties of $\bm{A}_{g}$, the details of which have been omitted. ∎

######  Lemma S8.10.

Suppose the assumptions of Lemma S8.8 hold and let
$\tilde{\bm{M}}=[n^{-1/2}\bm{X},\tilde{\bm{C}}]$. Then for any $a>0$, define
$\tilde{\bm{W}}_{g,a}=\operatorname{diag}[w_{g1}1\\{w_{g1}>a\\},\ldots,w_{gn}1\\{w_{gn}>a\\}]$.
Then there exists constants $c>0$ and $\eta_{a}>0$, the latter of which is a
decreasing function of $a$, and a random variable $z=O_{P}(n^{-1/2+\epsilon})$
such that

$\displaystyle(I_{K}+\bm{D}_{g})^{-1}\preceq$ $\displaystyle
I_{K}+c\bm{B}_{g}^{-\top}\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{M}}\tilde{\bm{M}}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}\bm{B}_{g}^{-1}-\eta_{a}\bm{B}_{g}^{-\top}\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}\bm{B}_{g}^{-1}$
$\displaystyle+\eta_{a}\bm{B}_{g}^{-\top}\tilde{\bm{z}}_{u}^{\top}\tilde{\bm{W}}_{g,a}\tilde{\bm{z}}_{u}\bm{B}_{g}^{-1}+zI_{K}$

###### Proof.

We assume $n^{-1}\bm{X}^{\top}\bm{X}=I_{d}$ without loss of generality. Then
we can express $\bm{D}_{g}$ as

$\displaystyle\bm{D}_{g}=$
$\displaystyle\bm{B}_{g}^{-T}\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}^{1/2}\bm{N}_{g}\bm{W}_{g}^{1/2}\tilde{\bm{z}}_{u}\bm{B}_{g}^{-1}$
$\displaystyle\bm{N}_{g}=$ $\displaystyle
I_{n}-\bm{W}_{g}^{1/2}\bm{X}(\bm{X}^{\top}\bm{W}_{g}\bm{X})\bm{X}^{\top}-\tilde{\bm{P}}_{g}\bm{W}_{g}^{1/2}\tilde{\bm{C}}\bm{A}_{g}^{-1}\tilde{\bm{C}}^{\top}\bm{W}_{g}^{1/2}\tilde{\bm{P}}_{g}$
$\displaystyle\tilde{\bm{P}}_{g}=$ $\displaystyle
I_{n}-\bm{W}_{g}^{1/2}\bm{X}(\bm{X}^{\top}\bm{W}_{g}\bm{X})\bm{X}^{\top}.$

To simplify the expression for $\bm{N}_{g}$, we first see that

$\displaystyle\tilde{\bm{P}}_{g}\bm{W}_{g}^{1/2}\tilde{\bm{C}}=\bm{W}_{g}^{1/2}\tilde{\bm{C}}-(n^{-1/2}\bm{W}_{g}^{1/2}\bm{X})(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X})\\{n^{-1}\bm{X}^{\top}(\bm{W}_{g}-I_{n})\tilde{\bm{C}}\\}.$

Corollary S8.1 can then be used to show that

$\displaystyle\max_{g\in[p]}\lVert
n^{-1/2}\bm{W}_{g}^{1/2}\bm{X}\rVert_{2},\,\max_{g\in[p]}\lVert
n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X}\rVert_{2}\leq 1+O_{P}(n^{-1/2+\epsilon})$
$\displaystyle\max_{g\in[p]}\lVert
n^{-1}\bm{X}^{\top}(\bm{W}_{g}-I_{n})\tilde{\bm{C}}\rVert_{2}=O_{P}(n^{-1/2+\epsilon}),$

which implies for $\tilde{\bm{X}}=n^{-1/2}\bm{X}$,

$\displaystyle\max_{g\in[p]}\lVert\tilde{\bm{P}}_{g}\bm{W}_{g}^{1/2}\tilde{\bm{C}}\bm{A}_{g}^{-1}\tilde{\bm{C}}^{\top}\bm{W}_{g}^{1/2}\tilde{\bm{P}}_{g}-\bm{W}_{g}^{1/2}\tilde{\bm{C}}\bm{A}_{g}^{-1}\tilde{\bm{C}}^{\top}\bm{W}_{g}^{1/2}\rVert_{2}=O_{P}(n^{-1/2+\epsilon})$
$\displaystyle\max_{g\in[p]}\lVert\bm{W}_{g}^{1/2}\bm{X}(\bm{X}^{\top}\bm{W}_{g}\bm{X})\bm{X}^{\top}-\bm{W}_{g}^{1/2}\tilde{\bm{X}}\tilde{\bm{X}}^{\top}\rVert_{2}=O_{P}(n^{-1/2+\epsilon}).$

Lemma S8.9 can then be used to simplify show

$\displaystyle\lVert\bm{W}_{g}^{1/2}\tilde{\bm{C}}\bm{A}_{g}^{-1}\tilde{\bm{C}}^{\top}\bm{W}_{g}^{1/2}-\bm{W}_{g}^{1/2}\tilde{\bm{C}}\tilde{\bm{C}}^{\top}\bm{W}_{g}^{1/2}\rVert_{2}=O_{P}(n^{-1/2+\epsilon}).$

Putting this all together implies for
$\tilde{\bm{M}}=[\tilde{\bm{X}},\tilde{\bm{C}}]$,

$\displaystyle\max_{g\in[p]}\lVert\bm{N}_{g}-(I_{n}-\bm{W}_{g}^{1/2}\tilde{\bm{M}}\tilde{\bm{M}}^{\top}\bm{W}_{g}^{1/2})\rVert_{2}=O_{P}(n^{-1/2+\epsilon}).$

Therefore, $\bm{D}_{g}$ satisfies

$\displaystyle\max_{g\in[p]}\lVert\bm{D}_{g}-\bm{B}_{g}^{-\top}\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}^{1/2}(I_{n}-\bm{W}_{g}^{1/2}\tilde{\bm{M}}\tilde{\bm{M}}^{\top}\bm{W}_{g}^{1/2})\bm{W}_{g}^{1/2}\tilde{\bm{z}}_{u}\bm{B}_{g}^{-1}\rVert_{2}=O_{P}(n^{-1/2+\epsilon}),$

where for some constant $c>0$

$\displaystyle\max_{g\in[p]}\lVert\bm{B}_{g}^{-\top}\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{M}}\tilde{\bm{M}}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}\bm{B}_{g}^{-1}\rVert_{2}\leq\delta^{2}c\\{1+O_{P}(n^{-1/2})\\}.$

Therefore, there exists a constant $\eta_{1}>0$ and random variable
$z=O_{P}(n^{-1/2+\epsilon})$ that does not depend on $g$ such that

$\displaystyle(I_{K}+\bm{D}_{g})^{-1}\preceq(I_{K}+\bm{B}_{g}^{-\top}\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}\bm{B}_{g}^{-1})^{-1}+\eta_{1}\bm{B}_{g}^{-\top}\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{M}}\tilde{\bm{M}}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}\bm{B}_{g}^{-1}+zI_{K}.$

Next, let $a>0$ be a constant and define
$\bar{\bm{W}}_{g,a}=\operatorname{diag}[w_{g1}1\\{w_{g1}\leq
a\\},\ldots,w_{gn}1\\{w_{gn}\leq a\\}]$. Then for some constant $\eta_{a}>0$
that is a decreasing function of $a$,

$\displaystyle(I_{K}+\bm{B}_{g}^{-\top}\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}\bm{B}_{g}^{-1})^{-1}\preceq(I_{K}+\bm{B}_{g}^{-\top}\tilde{\bm{z}}_{u}^{\top}\bar{\bm{W}}_{g,a}\tilde{\bm{z}}_{u}\bm{B}_{g}^{-1})^{-1}\preceq
I_{K}-\eta_{a}\bm{B}_{g}^{-\top}\tilde{\bm{z}}_{u}^{\top}\bar{\bm{W}}_{g,a}\tilde{\bm{z}}_{u}\bm{B}_{g}^{-1},$

which completes the proof. ∎

Returning to the proof of Lemma S8.8, let $a$, $\tilde{\bm{W}}_{g,a}$, and
$\tilde{\bm{M}}$ be as given in the statement of Lemma S8.10 and define

$\displaystyle\tilde{\bm{B}}_{g}=\bm{A}_{g}^{-1/2}\bm{B}_{g}=\bm{v}_{u}+\bm{A}_{g}^{-1}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{z}}_{u}$
$\displaystyle\bm{S}_{g}=\bm{B}_{g}^{-\top}\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{M}}\tilde{\bm{M}}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}\bm{B}_{g}^{-1}$
$\displaystyle\bm{R}_{g}=\bm{B}_{g}^{-\top}\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}\bm{B}_{g}^{-1},\quad\bm{R}_{g,a}=\bm{B}_{g}^{-\top}\tilde{\bm{z}}_{u}^{\top}\tilde{\bm{W}}_{g,a}\tilde{\bm{z}}_{u}\bm{B}_{g}^{-1}.$

Then for constants $c$ and $\eta_{a}$ as defined in the statement of Lemma
S8.10, Lemma S8.10 implies the objective can be lower bounded as

$\displaystyle\begin{aligned}
f_{1g}(\bm{C})-f_{1g}(\bm{U})\geq&\eta_{a}\operatorname{Tr}\\{\bm{R}_{g}(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}})^{1/2}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}})^{1/2}\\}\\\
&-\eta_{a}\operatorname{Tr}\\{\bm{R}_{g,a}(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}})^{1/2}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}})^{1/2}\\}\\\
&-c\operatorname{Tr}\\{\bm{S}_{g}(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}})^{1/2}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}})^{1/2}\\}+O_{P}(\lambda
n^{-1/2+\epsilon}),\end{aligned}$ (S8.9)

where the error term $O_{P}(\lambda n^{-1/2+\epsilon})$ is uniform over
$g\in[p]$. For the third term in (S8.9),

$\displaystyle\bm{M}^{(1)}_{g}$
$\displaystyle=\operatorname{Tr}\\{\bm{S}_{g}(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}})^{1/2}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}})^{1/2}\\}\leq\tilde{\bm{\ell}}_{g}^{\top}\tilde{\bm{\ell}}_{g}\operatorname{Tr}\\{\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{M}}\tilde{\bm{M}}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}(\tilde{\bm{B}}_{g}^{\top}\tilde{\bm{B}}_{g})^{-1}\\}$
$\displaystyle\leq
c\\{1+O_{P}(n^{-1/2+\epsilon})\\}\tilde{\bm{\ell}}_{g}^{\top}\tilde{\bm{\ell}}_{g}\sum_{k=1}^{K}\tilde{\bm{z}}_{u_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}}^{\top}\bm{W}_{g}\tilde{\bm{M}}\tilde{\bm{M}}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}}$

for some constant $c>0$ that does not depend on $n$ or $p$, where the second
inequality holds by Lemma S8.9 for $\delta$ small enough. Note that the by
Lemma S8.9, the $O_{P}(n^{-1/2+\epsilon})$ term is uniform over $g\in[p]$.
Define $s_{g}^{2}=\tilde{\bm{\ell}}_{g}^{\top}\tilde{\bm{\ell}}_{g}\leq
c\lambda\\{1+O_{P}(n^{-1/2})\\}$, where the error is uniform over $g\in[p]$.
Then

$\displaystyle(\lambda p)^{-1}\sum_{g=1}^{p}\bm{M}^{(1)}_{g}$
$\displaystyle\leq
c\\{1+O_{P}(n^{-1/2+\epsilon})\\}\sum_{k=1}^{K}\tilde{\bm{z}}_{u_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}}^{\top}\left\\{(\lambda
p)^{-1}\sum_{g=1}^{p}s_{g}^{2}\bm{W}_{g}\tilde{\bm{M}}\tilde{\bm{M}}^{\top}\bm{W}_{g}\right\\}\tilde{\bm{z}}_{u_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}}$
$\displaystyle=c\\{1+O_{P}(n^{-1/2+\epsilon})\\}\sum_{k=1}^{K}\sum_{r=1}^{d+K}(\lambda
p)^{-1}\tilde{\bm{z}}_{u_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}}^{\top}\sum_{g=1}^{p}s_{g}^{2}\bm{W}_{g}\tilde{\bm{M}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}r}\tilde{\bm{M}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}r}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}}.$

We see that

$\displaystyle(\lambda
p)^{-1}\tilde{\bm{z}}_{u_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}}^{\top}\sum_{g=1}^{p}s_{g}^{2}\bm{W}_{g}\tilde{\bm{M}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}r}\tilde{\bm{M}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}r}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}}=$
$\displaystyle
p^{-1}\tilde{\bm{z}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}^{\top}\bm{G}\bm{S}\bm{G}^{\top}\tilde{\bm{z}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}$
$\displaystyle\bm{G}=$
$\displaystyle[\bm{G}_{1}\cdots\bm{G}_{p}]\in\mathbb{R}^{n\times
p},\quad\bm{G}_{g}=\bm{W}_{g}\tilde{\bm{M}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}r}$
$\displaystyle\bm{S}=$
$\displaystyle\operatorname{diag}(s_{1}^{2}/\lambda,\ldots,s_{p}^{2}/\lambda),$

where by Cauchy-Schwarz and the fact that
$\left\lVert\bm{S}\right\rVert_{2}\leq c\\{1+O_{P}(n^{-1/2})\\}$ for some
constant $c>0$,

$\displaystyle 0\leq(\lambda
p)^{-1}\tilde{\bm{z}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}^{\top}\sum_{g=1}^{p}s_{g}^{2}\bm{W}_{g}\tilde{\bm{M}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}r}\tilde{\bm{M}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}r}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}}\leq
c\\{1+O_{P}(n^{-1/2})\\}(p^{-1}\tilde{\bm{z}}_{u_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}}^{\top}\bm{G}\bm{G}^{\top}\tilde{\bm{z}}_{u_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}})$

for some constant $c>0$. Let $\tilde{\bm{S}}=p^{-1}\bm{G}\bm{G}^{\top}$. Then

$\displaystyle\begin{aligned}
\tilde{\bm{S}}_{ij}=&\tilde{\bm{M}}_{ir}\tilde{\bm{M}}_{jr}p^{-1}\sum_{g=1}^{p}(w_{gi}-1)(w_{gj}-1)+\tilde{\bm{M}}_{ir}\tilde{\bm{M}}_{jr}p^{-1}\sum_{g=1}^{p}w_{gi}+\tilde{\bm{M}}_{ir}\tilde{\bm{M}}_{jr}p^{-1}\sum_{g=1}^{p}w_{gj}\\\
&-\tilde{\bm{M}}_{ir}\tilde{\bm{M}}_{jr}.\end{aligned}$ (S8.10)

Since
$\tilde{\bm{z}}_{u_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}}^{\top}\tilde{\bm{M}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}r}=0$
for all $k\in[K]$ and $r\in[d+K]$, the last three terms in (S8.10) are
nullified, implying we need only study the first term. We then have that for
$\bm{M}=\in\mathbb{R}^{n\times p}$ such that $\bm{M}_{ig}=w_{gi}-1$,
$\tilde{\bm{S}}=p^{-1}\operatorname{diag}(\tilde{\bm{M}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}r})\bm{M}\bm{M}^{\top}\operatorname{diag}(\tilde{\bm{M}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}r})$.
By Lemma S8.6, $\lVert p^{-1/2}\bm{M}\rVert_{2}=O_{P}(n^{\epsilon})$ for an
arbitrarily small constant $\epsilon>0$. Therefore,
$\lVert\tilde{\bm{S}}\rVert_{2}=O_{P}(n^{\epsilon}\max_{i\in[n]}\tilde{\bm{M}}_{ir}^{2})=O_{P}(n^{-1+\epsilon})$,
which implies $(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{M}_{g}^{(1)}=O_{P}(\delta^{2}n^{-1+\epsilon})$.

We next consider the first term in (S8.9). Here,

$\displaystyle\operatorname{Tr}\\{\bm{R}_{g}(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}})^{1/2}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}})^{1/2}\\}=\operatorname{Tr}(\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}\tilde{\bm{B}}_{g}^{-1}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\tilde{\bm{B}}_{g}^{-T}).$

Since $f_{1g}(\bm{U})$ in (S8.4) only depends on $\operatorname{Im}(\bm{U})$,
it suffices to assume $\lVert I_{K}-\bm{v}_{u}\rVert_{2}=O(\delta^{2})$. Let
$\bm{\Delta}_{g}=I_{K}-\tilde{\bm{B}}_{g}^{-1}$, where an identical analysis
to that used to prove (S8.8) in Lemma S8.9 can be used to show that
$\max_{g\in[p]}\lVert\bm{\Delta}_{g}\rVert_{2}\leq
c(\delta+\delta^{2})\\{1+O_{P}(n^{-1/2+\epsilon})\\}$. Next,

$\displaystyle\operatorname{Tr}(\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}\tilde{\bm{B}}_{g}^{-1}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\tilde{\bm{B}}_{g}^{-T})=$
$\displaystyle\operatorname{Tr}(\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top})-\operatorname{Tr}\\{\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}(\bm{\Delta}_{g}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\tilde{\bm{B}}_{g}^{-\top}+\tilde{\bm{B}}_{g}^{-1}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\bm{\Delta}_{g}^{\top})\\}$
$\displaystyle+\operatorname{Tr}(\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}\bm{\Delta}_{g}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\bm{\Delta}_{g}^{\top})$
$\displaystyle\geq$
$\displaystyle\operatorname{Tr}(\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top})-\operatorname{Tr}\\{\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}(\bm{\Delta}_{g}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\tilde{\bm{B}}_{g}^{-\top}+\tilde{\bm{B}}_{g}^{-1}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\bm{\Delta}_{g}^{\top})\\}.$

For $\bm{R}=n^{-1}\bm{C}^{\top}P_{X}^{\perp}\bm{C}$,

$\displaystyle(\lambda
p)^{-1}\sum_{g=1}^{p}\operatorname{Tr}(\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top})=$
$\displaystyle(\lambda
p)^{-1}\sum_{g=1}^{p}n\operatorname{Tr}(\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}\bm{R}^{1/2}\bm{\ell}_{g}\bm{\ell}_{g}^{\top}\bm{R}^{1/2})$
$\displaystyle=$ $\displaystyle(\lambda
p)^{-1}\sum_{g=1}^{p}n\operatorname{Tr}(\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}\bm{\ell}_{g}\bm{\ell}_{g}^{\top})+O_{P}(\delta^{2}n^{-1/2+\epsilon})$
$\displaystyle(\lambda
p)^{-1}\sum_{g=1}^{p}n\operatorname{Tr}(\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}\bm{\ell}_{g}\bm{\ell}_{g}^{\top})\geq$
$\displaystyle\lVert\tilde{\bm{z}}_{u}\rVert_{F}^{2}\lambda_{K}/\lambda+\sum\limits_{r,s=1}^{K}\sum\limits_{i=1}^{n}\tilde{\bm{z}}_{u_{ir}}\tilde{\bm{z}}_{u_{is}}\underbrace{p^{-1}\sum\limits_{g=1}^{p}(n\bm{\ell}_{gr}\bm{\ell}_{gs}/\lambda)(w_{gi}-1)}_{=x_{irs}}.$

Since $\\{w_{gi}-1\\}_{g\in[p]}$ are mean 0 and independent conditional on
$\bm{C}$ and $\max_{g\in[p]}\lvert
n\bm{\ell}_{gr}\bm{\ell}_{gs}/\lambda\rvert\leq c$ for some constant $c>0$,
Corollary S8.1 implies

$\displaystyle\max_{\begin{subarray}{c}i\in[n],\,r,s\in[K]\end{subarray}}\left\lvert
x_{irs}\right\rvert=O_{P}(n^{-1/2+\epsilon}).$

Lastly,
$\max_{g\in[p]}s_{g}^{-2}\lVert\bm{\Delta}_{g}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\tilde{\bm{B}}_{g}^{-\top}+\tilde{\bm{B}}_{g}^{-1}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\bm{\Delta}_{g}^{\top}\rVert_{2}\leq
c\delta\\{1+O_{P}(n^{-1/2+\epsilon})\\}$ for some constant $c>0$ and $\delta$
small enough, meaning

$\displaystyle\lvert\operatorname{Tr}\\{\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}(\bm{\Delta}_{g}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\tilde{\bm{B}}_{g}^{-\top}+\tilde{\bm{B}}_{g}^{-1}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\bm{\Delta}_{g}^{\top})\\}\rvert\leq
c\delta\\{1+O_{P}(n^{-1/2+\epsilon})\\}s_{g}^{2}\operatorname{Tr}(\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u})$

for some constant $c>0$. Therefore,

$\displaystyle\lvert(\lambda
p)^{-1}\sum\limits_{g=1}^{p}\operatorname{Tr}\\{\tilde{\bm{z}}_{u}^{\top}\bm{W}_{g}\tilde{\bm{z}}_{u}(\bm{\Delta}_{g}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\tilde{\bm{B}}_{g}^{-\top}+\tilde{\bm{B}}_{g}^{-1}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\bm{\Delta}_{g}^{\top})\\}\rvert\leq
c\delta^{3}\\{1+O_{P}(n^{-1/2+\epsilon})\\}$

for some constant $c>0$.

We lastly consider the second term in (S8.9). For $a>0$ as defined in the
statement of Lemma S8.10,
$\operatorname{\mathbb{E}}[w_{gi}1\\{w_{gi}>a\\}]\leq\epsilon_{a}$. for some
constant $\epsilon_{a}\geq 0$ that is a non-increasing function of $a$, and
can be made arbitrarily small. Then there exists a constant $c>0$ and random
variable $z=O_{P}(n^{-1/2+\epsilon})$ that does not depend on $g$ such that

$\displaystyle\bm{M}_{g}^{(2)}=$
$\displaystyle\operatorname{Tr}\\{\bm{R}_{g,a}(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}})^{1/2}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}})^{1/2}\\}\leq(c+z)\lambda\operatorname{Tr}(\tilde{\bm{z}}_{u}^{\top}\tilde{\bm{W}}_{g,a}\tilde{\bm{z}}_{u})\leq\delta^{2}(c+z)\lambda\epsilon_{a}$
$\displaystyle+(c+z)\lambda\operatorname{Tr}[\tilde{\bm{z}}_{u}^{\top}\\{\tilde{\bm{W}}_{g,a}-\operatorname{\mathbb{E}}(\tilde{\bm{W}}_{g,a})\\}\tilde{\bm{z}}_{u}].$

We see that

$\displaystyle
p^{-1}\sum_{g=1}^{p}\operatorname{Tr}[\tilde{\bm{z}}_{u}^{\top}\\{\tilde{\bm{W}}_{g,a}-\operatorname{\mathbb{E}}(\tilde{\bm{W}}_{g,a})\\}\tilde{\bm{z}}_{u}]=\sum_{k=1}^{K}\sum_{i=1}^{n}\tilde{\bm{z}}_{u_{ik}}^{2}p^{-1}\sum_{g=1}^{p}(w_{gi}1\\{w_{gi}>a\\}-\operatorname{\mathbb{E}}[w_{gi}1\\{w_{gi}>a\\}]),$

where

$\displaystyle\max_{i\in[n]}\lvert
p^{-1}\sum_{g=1}^{p}(w_{gi}1\\{w_{gi}>a\\}-\operatorname{\mathbb{E}}[w_{gi}1\\{w_{gi}>a\\}])\rvert=O_{P}(n^{-1/2+\epsilon}).$

Choosing $a>0$ large enough, and therefore $\epsilon_{a}\geq 0$ small enough,
thus completes the proof. ∎

######  Lemma S8.11.

Suppose Assumption S8.4 holds and let
$\Omega_{\delta}=\\{\bm{U}\in\mathbb{R}^{n\times
K}:\bm{U}^{\top}\bm{U}=I_{K},\bm{U}^{\top}\bm{X}=\bm{0},\left\lVert
P_{\bm{U}}-P_{\tilde{\bm{C}}}\right\rVert_{2}\leq\delta\\}$. Then for all
$\delta>0$ sufficiently small, there exists a constant $c>0$ such that for all
$\epsilon\in(0,1/2)$,
$\sup_{\bm{U}\in\Omega_{\delta}}\max_{g\in[p]}\lVert(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\rVert_{2}\leq
c+O_{P}(n^{-1/2+\epsilon})$.

###### Proof.

For $\bm{v}_{u}$ and $\bm{z}_{u}$ as defined in Lemma S8.3, Lemma S8.3 implies
that for some constant $c>0$,

$\displaystyle\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U}\succeq(1-\delta^{2}c)I_{K}+\bm{v}_{u}^{\top}(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}-I_{K})\bm{v}_{u}+\bm{v}_{u}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{Q}\bm{z}_{u}+(\bm{v}_{u}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{Q}\bm{z}_{u})^{\top}.$

By Lemma S8.9 and the proof of (S8.8) in Lemma S8.9,

$\displaystyle\max_{g\in[p]}\lVert\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}-I_{K}\rVert_{2}=O_{P}(n^{-1/2+\epsilon})$
$\displaystyle\sup_{\bm{U}\in\Omega_{\delta}}\max_{g\in[p]}\lVert\bm{v}_{u}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{Q}\bm{z}_{u}\rVert_{2}\leq\\{c+O_{P}(n^{-1/2+\epsilon})\\}\sup_{\bm{U}\in\Omega_{\delta}}\left\lVert\bm{z}_{u}\right\rVert_{2}$

for some constant $c>0$. Since
$\sup_{\bm{U}\in\Omega_{\delta}}\left\lVert\bm{z}_{u}\right\rVert_{2}=O(\delta)$
by Lemma S8.3, this completes the proof. ∎

######  Lemma S8.12.

Define $f_{3}(\bm{U})=(\lambda
p)^{-1}\sum_{g=1}^{p}\operatorname{Tr}\\{(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\bm{e}_{g}^{\top}\bm{P}_{g}^{\perp}\bm{U}\\}$
and suppose Assumption S8.4 holds. Then for any constant $\epsilon\in(0,1/2)$,

$\displaystyle\sup_{\bm{U}_{\delta}\in\Omega}f_{3}(\bm{U})=O_{P}(\lambda^{-1+\epsilon}),\quad\Omega_{\delta}=\\{\bm{U}\in\mathbb{R}^{n\times
K}:\bm{U}^{\top}\bm{U}=I_{K},\bm{U}^{\top}\bm{X}=\bm{0},\left\lVert
P_{\bm{U}}-P_{\tilde{\bm{C}}}\right\rVert_{2}\leq\delta\\}$

for all $\delta>0$ sufficiently small.

###### Proof.

Let $\epsilon\in(0,1/2)$ be an arbitrarily small constant. For any
$\bm{U}\in\Omega_{\delta}$, there exists a constant $c>0$ such that

$\displaystyle f_{3}(\bm{U})\leq c\\{1+o_{P}(1)\\}(\lambda
p)^{-1}\sum_{g=1}^{p}\operatorname{Tr}\\{\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\bm{e}_{g}^{\top}\bm{P}_{g}^{\perp}\bm{U}\\}$
$\displaystyle=$ $\displaystyle c\\{1+o_{P}(1)\\}(\lambda
p)^{-1}\sum_{g=1}^{p}\operatorname{Tr}\\{\bm{U}^{\top}\bm{W}_{g}\bm{e}_{g}\bm{e}_{g}^{\top}\bm{W}_{g}\bm{U}\\}$
$\displaystyle-2c\\{1+o_{P}(1)\\}\underbrace{(\lambda
p)^{-1}\sum_{g=1}^{p}\operatorname{Tr}\\{\bm{U}^{\top}\bm{W}_{g}\bm{X}(\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{e}_{g}\bm{e}_{g}^{\top}\bm{W}_{g}\bm{U}\\}}_{=S(\bm{U})}$
$\displaystyle+c\\{1+o_{P}(1)\\}\underbrace{(\lambda
p)^{-1}\sum_{g=1}^{p}\operatorname{Tr}\\{\bm{U}^{\top}\bm{W}_{g}\bm{X}(\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{e}_{g}\bm{e}_{g}^{\top}\bm{W}_{g}\bm{X}(\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{U}\\}}_{=T(\bm{U})}$

by Lemma S8.11 for $\delta>0$ sufficiently small. By Lemma S8.7,

$\displaystyle\sup_{\bm{U}\in\Omega_{\delta}}(\lambda
p)^{-1}\sum_{g=1}^{p}\operatorname{Tr}\\{\bm{U}^{\top}\bm{W}_{g}\bm{e}_{g}\bm{e}_{g}^{\top}\bm{W}_{g}\bm{U}\\}=O_{P}(\lambda^{-1+\epsilon}).$

Since $f_{3}(\bm{U})$ only depends on $\bm{X}$ through
$\operatorname{Im}(\bm{X})$, it suffices to assume
$n^{-1}\bm{X}^{\top}\bm{X}=I_{d}$, where by (a) of Assumption S8.4, the
entries of $\bm{X}$ are uniformly bounded from above and below. Define
$\bm{\Delta}_{g}=I_{d}-(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}$. By
Corollary S8.1,
$\max_{g\in[p]}\lVert\bm{\Delta}_{g}\rVert_{2}=O_{P}(n^{-1/2+\epsilon})$ and

$\displaystyle S(\bm{U})=$ $\displaystyle(\lambda
p)^{-1}\operatorname{Tr}\left(\bm{U}^{\top}\sum_{s=1}^{d}\bm{A}_{s}\bm{U}\right)+(\lambda
p)^{-1}\operatorname{Tr}\left(\bm{U}^{\top}\sum_{r,s=1}^{d}\bm{B}_{rs}\bm{U}\right)$
$\displaystyle\bm{A}_{s}=$
$\displaystyle\operatorname{diag}(\bm{X}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}s})\bm{W}^{\top}\operatorname{diag}(n^{-1}\bm{X}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}s}^{\top}\bm{W}_{1}\bm{e}_{1},\ldots,n^{-1}\bm{X}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}s}^{\top}\bm{W}_{p}\bm{e}_{p})\tilde{\bm{E}},\quad
s\in[d]$ $\displaystyle\bm{B}_{rs}=$
$\displaystyle\operatorname{diag}(\bm{X}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}r})\bm{W}^{\top}\operatorname{diag}(n^{-1}\bm{\Delta}_{1_{rs}}\bm{X}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}s}^{\top}\bm{W}_{1}\bm{e}_{1},\ldots,n^{-1}\bm{\Delta}_{p_{rs}}\bm{X}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}s}^{\top}\bm{W}_{p}\bm{e}_{p})\tilde{\bm{E}},\quad
r,s\in[d]$ $\displaystyle\bm{W}_{gi}=$ $\displaystyle
w_{gi}-1,\quad\tilde{\bm{E}}_{gi}=\bm{e_{g_{i}}}w_{gi},\quad g\in[p];i\in[n].$

Since the entries of $\bm{X}$ are uniformly bounded,
$\lVert\operatorname{diag}(\bm{X}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}s})\rVert_{2}=O(1)$
for all $s\in[d]$. By Lemmas S8.6 and S8.7, $\lVert
p^{-1/2}\bm{W}\rVert_{2},\lVert
p^{-1/2}\tilde{\bm{E}}\rVert_{2}=O_{P}(n^{\epsilon})$, and by Corollary S8.1,

$\displaystyle\lVert\operatorname{diag}(n^{-1}\bm{X}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}s}^{\top}\bm{W}_{1}\bm{e}_{1},\ldots,n^{-1}\bm{X}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}s}^{\top}\bm{W}_{p}\bm{e}_{p})\rVert_{2}=O_{P}(n^{-1/2+\epsilon}).$

Putting this all together implies
$\sup_{\bm{U}\in\Omega}S(\bm{U})=O_{P}(\lambda^{-1}n^{-1/2+\epsilon})$. An
identical analysis can be used to show that
$\sup_{\bm{U}\in\Omega}T(\bm{U})=O_{P}(\lambda^{-1}n^{-1+\epsilon})$, which
completes the proof. ∎

######  Lemma S8.13.

Define $f_{2}(\bm{U})=(\lambda
p)^{-1}\sum_{g=1}^{p}\operatorname{Tr}\\{(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\bm{U}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\tilde{\bm{\ell}}_{g}\bm{e}_{g}^{\top}\bm{P}_{g}^{\perp}\bm{U}\\}$.
Then under the assumptions of Lemma S8.12,
$\sup_{\bm{U}\in\Omega_{\delta}}\lvert
f_{2}(\bm{U})\rvert=O_{P}(\lambda^{-1/2+\epsilon})$ for any constant
$\epsilon\in(0,1/2)$ and $\delta>0$ sufficiently small.

###### Proof.

Let $\bm{U}\in\Omega$ and define $a_{g_{rs}}$ to be the $r,s$ element of
$(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1/2}$ for $(r,s)\in[K]\times[K]$.
Note that $\max_{g\in[p]}\lvert a_{g_{rs}}\rvert\leq c\\{1+o_{P}(1)\\}$ for
some constant $c>0$ by Lemma S8.11. Then for
$\bar{\bm{\ell}}_{g}=\lambda^{-1/2}\tilde{\bm{\ell}}$, where
$\max_{g\in[p]}\bar{\bm{\ell}}_{g}\leq c\\{1+o_{P}(1)\\}$ for some constant
$c>0$,

$\displaystyle f_{2}(\bm{U})=$
$\displaystyle\lambda^{-1/2}\sum_{r,s=1}^{K}\bm{U}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}r}^{\top}\bm{A}^{\top}\bm{B}_{rs}\bm{U}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}s},\quad\bm{A}=p^{-1/2}\begin{pmatrix}\bar{\bm{\ell}}_{1}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{1}^{\perp}\\\
\vdots\\\
\bar{\bm{\ell}}_{p}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{p}^{\perp}\end{pmatrix},\quad\bm{B}_{rs}=p^{-1/2}\begin{pmatrix}a_{1_{rs}}\bm{e}_{1}^{\top}\bm{P}_{1}^{\perp}\\\
\vdots\\\ a_{p_{rs}}\bm{e}_{p}^{\top}\bm{P}_{p}^{\perp}\end{pmatrix},$

where $\lVert\bm{A}\rVert_{2},\lVert\bm{B}_{rs}\rVert_{2}=O_{P}(n^{\epsilon})$
by the proofs of Lemmas S8.8 and S8.12. ∎

######  Corollary S8.2.

Suppose the assumptions of Lemma S8.12 hold and let $\Omega_{\delta}$ be as
defined in the statement of Lemma S8.12. Then for $f$ defined in (S8.1),
$\hat{\bm{C}}=\operatorname*{argmax}_{\bm{U}\in\Omega_{\delta}}f(\bm{U})$, and
$\delta>0$ sufficiently small, there exists a constant $\eta\in(0,1/4)$ such
that $\lVert P_{\hat{\bm{C}}}-P_{\tilde{\bm{C}}}\rVert_{2}=O_{P}(n^{-\eta})$
as $n,p\to\infty$.

###### Proof.

This is a direct consequence of Lemmas S8.8, S8.12, and S8.13. ∎

###  S8.4 Properties and rate of convergence of $\hat{\bm{C}}$

Here we study the properties and rate of convergence of $\hat{\bm{C}}$. To do
so, we use the decomposition discussed in Lemma S8.3, where any
$\bm{U}\in\mathbb{R}^{n\times K}$ such that $\bm{U}^{\top}\bm{U}=I_{K}$ can be
expressed as $\bm{U}=\tilde{\bm{C}}\bm{v}_{u}+\bm{Q}\bm{z}_{u}$, where the
columns of $\bm{Q}\in\mathbb{R}^{n\times(n-K)}$ form an orthonormal basis for
$\ker(\tilde{\bm{C}}^{\top})$, $\bm{v}_{u},\bm{z}_{u}$ depend on $\bm{U}$, and
$\bm{v}_{u}^{\top}\bm{v}_{u}+\bm{z}_{u}^{\top}\bm{z}_{u}=I_{K}$. We can
therefore write $\hat{\bm{C}}$ defined in the statement of Corollary S8.2 as
$\hat{\bm{C}}=\tilde{\bm{C}}\hat{\bm{v}}+\bm{Q}\hat{\bm{z}}$, where to
understand the properties of $\hat{\bm{C}}$, we need only determine
$\hat{\bm{v}}$ and $\hat{\bm{z}}$.

Define
$\tilde{f}\\{(\bm{v}^{\top}\,\bm{z}^{\top})^{\top}\\}=f(\tilde{\bm{C}}\bm{v}+\bm{Q}\bm{z})$,
where $f$ is as defined in (S8.1). Then for
$\bm{U}=\tilde{\bm{C}}\bm{v}_{u}+\bm{Q}\tilde{\bm{z}}_{u}$,

$\displaystyle\begin{aligned}
&\tilde{\bm{s}}\\{(\bm{v}_{u}^{\top}\,\bm{z}_{u}^{\top})^{\top}\\}=\begin{pmatrix}\tilde{\bm{C}}^{\top}\\\
\bm{Q}^{\top}\end{pmatrix}\nabla_{\bm{U}}f(\bm{U})\\\
=&(p\lambda)^{-1}\sum_{g=1}^{p}\begin{pmatrix}\tilde{\bm{C}}^{\top}\\\
\bm{Q}^{\top}\end{pmatrix}\\{\bm{P}_{g}^{\perp}-\bm{P}_{g}^{\perp}\bm{U}(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\bm{U}^{\top}\bm{P}_{g}^{\perp}\\}\bm{y}_{g}\bm{y}_{g}^{\top}\bm{P}_{g}^{\perp}\bm{U}(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1},\end{aligned}$
(S8.11)

where for any unitary matrix $\bm{v}\in\mathbb{R}^{K\times K}$,

$\displaystyle\begin{aligned}
&\tilde{\bm{s}}\\{(\bm{v}^{\top}\,\bm{0})^{\top}\\}=\begin{pmatrix}\bm{0}_{K\times
K}\\\
(p\lambda)^{-1}\sum_{g=1}^{p}\bm{Q}^{\top}\\{\bm{P}_{g}^{\perp}-\bm{P}_{g}^{\perp}\tilde{\bm{C}}(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}})^{-1}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\\}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}\end{pmatrix}\bm{v}\\\
&+\begin{pmatrix}\bm{0}_{K\times K}\\\
(p\lambda)^{-1}\sum_{g=1}^{p}\bm{Q}^{\top}\\{\bm{P}_{g}^{\perp}-\bm{P}_{g}^{\perp}\tilde{\bm{C}}(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}})^{-1}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\\}\bm{e}_{g}\bm{e}_{g}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}})^{-1}\end{pmatrix}\bm{v}.\end{aligned}$
(S8.12)

The Hessian can be expressed as

$\displaystyle\begin{aligned}
\tilde{\bm{H}}(\bm{U})=&\\{I_{K}\otimes(\tilde{\bm{C}}\,\bm{Q})^{\top}\\}\nabla_{\bm{U}}^{2}f(\bm{U})\\{I_{K}\otimes(\tilde{\bm{C}}\,\bm{Q})\\}\\\
=&(\lambda
p)^{-1}\sum_{g=1}^{p}\\{I_{K}\otimes(\tilde{\bm{C}}\,\bm{Q})^{\top}\\}(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\otimes\\{\bm{A}_{g}(\bm{U})\bm{y}_{g}\bm{y}_{g}^{\top}\bm{A}_{g}(\bm{U})\\}\\{I_{K}\otimes(\tilde{\bm{C}}\,\bm{Q})\\}\\\
&-(\lambda
p)^{-1}\sum_{g=1}^{p}\\{I_{K}\otimes(\tilde{\bm{C}}\,\bm{Q})^{\top}\\}\bm{B}_{g}(\bm{U})^{\top}\otimes\\{\bm{A}_{g}(\bm{U})\bm{y}_{g}\bm{y}_{g}^{\top}\bm{B}_{g}(\bm{U})\\}\bm{\Pi}\\{I_{K}\otimes(\tilde{\bm{C}}\,\bm{Q})\\}\\\
&-(\lambda
p)^{-1}\sum_{g=1}^{p}\\{I_{K}\otimes(\tilde{\bm{C}}\,\bm{Q})^{\top}\\}\\{\bm{B}_{g}(\bm{U})^{\top}\bm{y}_{g}\bm{y}_{g}^{\top}\bm{B}_{g}(\bm{U})\\}\otimes\bm{A}_{g}(\bm{U})\\{I_{K}\otimes(\tilde{\bm{C}}\,\bm{Q})\\}\\\
&-(\lambda
p)^{-1}\sum_{g=1}^{p}\\{I_{K}\otimes(\tilde{\bm{C}}\,\bm{Q})^{\top}\\}\\{\bm{B}_{g}(\bm{U})^{\top}\bm{y}_{g}\bm{y}_{g}^{\top}\bm{A}_{g}(\bm{U})\\}\otimes\bm{B}_{g}(\bm{U})\bm{\Pi}\\{I_{K}\otimes(\tilde{\bm{C}}\,\bm{Q})\\}\end{aligned}$
(S8.13)
$\displaystyle\bm{A}_{g}(\bm{U})=\bm{P}_{g}^{\perp}-\bm{P}_{g}^{\perp}\bm{U}(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\bm{U}^{\top}\bm{P}_{g}^{\perp},\quad\bm{B}_{g}(\bm{U})=\bm{P}_{g}^{\perp}\bm{U}(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1},$

where $\bm{\Pi}\in\mathbb{R}^{nK\times nK}$ is a permutation matrix that
satisfies
$\bm{\Pi}\operatorname{vec}(\bm{U})=\operatorname{vec}(\bm{U}^{\top})$ for
$\bm{U}\in\mathbb{R}^{n\times K}$. We next prove a series of lemmas that
facilitate understanding $\tilde{\bm{s}}$ and $\tilde{\bm{H}}$, and will lead
to an exact expression for $\hat{\bm{C}}-\tilde{\bm{C}}$.

######  Lemma S8.14 (First term in (S8.13)).

Define
$\bm{H}_{g}^{(1)}(\bm{U})=(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\otimes\\{\bm{A}_{g}(\bm{U})\bm{y}_{g}\bm{y}_{g}^{\top}\bm{A}_{g}(\bm{U})\\}$,
suppose Assumption S8.4 holds, let $\eta\in(0,1/2)$, and let $\hat{\bm{C}}$ be
as defined in Corollary S8.2. Then for $\delta=O_{P}(n^{-\eta})$ and any
constant $\epsilon\in(0,\eta)$,

$\displaystyle\sup_{\bm{U}\in\Omega_{\delta}}\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{H}_{g}^{(1)}(\bm{U})\rVert_{2}=O_{P}(n^{-2\eta+\epsilon}+\lambda^{-1+\epsilon}).$

###### Proof.

By Lemma S8.11, $(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\preceq
c\\{1+o_{P}(1)\\}I_{K}$ for some constant $c>0$. Therefore,

$\displaystyle(\lambda p)^{-1}\sum_{g=1}^{p}\bm{H}_{g}^{(1)}(\bm{U})\preceq
c\\{1+o_{P}(1)\\}I_{K}\otimes\left[(\lambda
p)^{-1}\sum_{g=1}^{p}\\{\bm{A}_{g}(\bm{U})\bm{y}_{g}\bm{y}_{g}^{\top}\bm{A}_{g}(\bm{U})\\}\right].$

First,

$\displaystyle\begin{aligned}
\bm{A}_{g}(\bm{U})\bm{y}_{g}=&\bm{A}_{g}(\bm{U})\tilde{\bm{C}}\tilde{\bm{\ell}}_{g}+\bm{A}_{g}(\bm{U})\bm{e}_{g}\\\
\bm{A}_{g}(\bm{U})\tilde{\bm{C}}\tilde{\bm{\ell}}_{g}=&\bm{P}_{g}^{\perp}\tilde{\bm{C}}\tilde{\bm{\ell}}_{g}-\bm{P}_{g}^{\perp}\bm{U}(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\bm{U}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\tilde{\bm{\ell}}_{g}\\\
\bm{A}_{g}(\bm{U})\bm{e}_{g}=&\bm{P}_{g}^{\perp}\bm{e}_{g}-\bm{P}_{g}^{\perp}\bm{U}(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}.\end{aligned}$
(S8.14)

By the proof of Lemma S8.12,

$\displaystyle\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{P}_{g}^{\perp}\bm{e}_{g}\bm{e}_{g}^{\top}\bm{P}_{g}^{\perp}\rVert_{2}=O_{P}(\lambda^{-1+\epsilon}).$

Next, since $\delta=O_{P}(n^{-\eta})$, it is straightforward to show that

$\displaystyle\sup_{\bm{U}}\max_{g\in[p]}\bm{U}^{T}(\bm{P}_{g}^{\perp})^{2}\bm{U}\leq
c\\{1+o_{P}(1)\\}$

for some constant $c>0$, which implies

$\displaystyle\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{P}_{g}^{\perp}\bm{U}(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\bm{e}_{g}^{\top}\bm{P}_{g}^{\perp}\bm{U}(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\bm{U}^{T}\bm{P}_{g}^{\perp}\rVert_{2}$
$\displaystyle\leq$ $\displaystyle(\lambda
p)^{-1}\sum_{g=1}^{p}\operatorname{Tr}\\{\bm{P}_{g}^{\perp}\bm{U}(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\bm{e}_{g}^{\top}\bm{P}_{g}^{\perp}\bm{U}(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\bm{U}^{T}\bm{P}_{g}^{\perp}\\}$
$\displaystyle=$ $\displaystyle(\lambda
p)^{-1}\sum_{g=1}^{p}\operatorname{Tr}\\{\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\bm{e}_{g}^{\top}\bm{P}_{g}^{\perp}\bm{U}(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\bm{U}^{T}(\bm{P}_{g}^{\perp})^{2}\bm{U}(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\\}$
$\displaystyle\leq$ $\displaystyle c\\{1+o_{P}(1)\\}(\lambda
p)^{-1}\sum_{g=1}^{p}\operatorname{Tr}\\{\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\bm{e}_{g}^{\top}\bm{P}_{g}^{\perp}\bm{U}\\}.$

where the $o_{P}(1)$ error term is uniform over all
$\bm{U}\in\Omega_{\delta}$. The proof of Lemma S8.12 shows that

$\displaystyle\sup_{\bm{U}\in\Omega_{\delta}}(\lambda
p)^{-1}\sum_{g=1}^{p}\operatorname{Tr}\\{\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\bm{e}_{g}^{\top}\bm{P}_{g}^{\perp}\bm{U}\\}=O_{P}(\lambda^{-1+\epsilon})$

and implies

$\displaystyle\sup_{\bm{U}\in\Omega_{\delta}}\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{A}_{g}(\bm{U})\bm{e}_{g}\bm{e}_{g}^{\top}\bm{A}_{g}(\bm{U})\rVert_{2}=O_{P}(\lambda^{-1+\epsilon}).$

For the remaining term in (S8.14), let
$\bm{U}=\tilde{\bm{C}}\bm{v}_{u}+\tilde{\bm{z}}_{u}$, where
$\tilde{\bm{z}}_{u}\in\ker([\bm{X},\tilde{\bm{C}}]^{\top})$ and
$\bm{v}_{u}^{\top}\bm{v}_{u}+\tilde{\bm{z}}_{u}^{\top}\tilde{\bm{z}}_{u}=I_{K}$.
By Lemma S8.3, $\lVert\bm{v}_{u}^{\top}\bm{v}_{u}-I_{K}\rVert_{2}\leq
c\delta^{2}$ and $\lVert\tilde{\bm{z}}_{u}\rVert_{2}\leq c\delta$ for some
constant $c>0$. Therefore, since $\delta=O_{P}(n^{-\eta})$ and by the proof of
(S8.8) of Lemma S8.9,

$\displaystyle\sup_{\bm{U}\in\Omega_{\delta}}\max_{g\in[p]}\lVert\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U}-\bm{v}_{u}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\bm{v}_{u}\rVert_{2}\leq
c\delta\\{1+o_{P}(1)\\}$

for some constant $c>0$. Therefore,

$\displaystyle(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\bm{U}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}=(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\bm{v}_{u}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}+(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\tilde{\bm{z}}_{u}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}$
$\displaystyle\Rightarrow\sup_{\bm{U}\in\Omega_{\delta}}\max_{g\in[p]}\lVert(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\bm{U}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}-\bm{v}_{u}^{-1}\rVert_{2}=O_{P}(n^{-\eta}).$

Consequently,

$\displaystyle\sup_{\bm{U}\in\Omega_{\delta}}\max_{g\in[p]}\lVert\bm{A}_{g}(\bm{U})\tilde{\bm{C}}\rVert_{2}\leq
O_{P}(n^{-\eta})+\sup_{\bm{U}\in\Omega_{\delta}}\max_{g\in[p]}\left\lVert\bm{P}_{g}^{\perp}\tilde{\bm{z}}_{u}\right\rVert_{2}=O_{P}(n^{-\eta+\epsilon})$

for any $\epsilon\in(0,\eta)$, which completes the proof. ∎

######  Lemma S8.15 (Third term in (S8.13)).

Suppose the assumptions of Lemma S8.14 hold, let $\eta\in(0,1/2)$, and let
$\Omega_{\delta}$ be as defined in Lemma S8.13. Then if
$\delta=O_{P}(n^{-\eta})$, there exists a unitary matrix
$\bm{v}=\bm{v}(\bm{U})\in\mathbb{R}^{K\times K}$ that depends on
$\bm{U}\in\Omega_{\delta}$ such that

$\displaystyle\sup_{\bm{U}\in\Omega_{\delta}}\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\\{\bm{B}_{g}(\bm{U})^{\top}\bm{y}_{g}\bm{y}_{g}^{\top}\bm{B}_{g}(\bm{U})\\}\otimes\bm{A}_{g}(\bm{U})-(\lambda^{-1}\bm{v}^{\top}\bm{\Lambda}\bm{v})\otimes\bm{Q}\bm{Q}^{\top}\rVert_{2}=O_{P}(n^{-\eta+\epsilon}+\lambda^{-1/2+\epsilon})$

for any constant $\epsilon>0$.

###### Proof.

We see that

$\displaystyle\begin{aligned}
\bm{B}_{g}(\bm{U})^{\top}\bm{y}_{g}\bm{y}_{g}^{\top}\bm{B}_{g}(\bm{U})=&\bm{B}_{g}(\bm{U})^{\top}\tilde{\bm{C}}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\tilde{\bm{C}}^{\top}\bm{B}_{g}(\bm{U})+\bm{B}_{g}(\bm{U})^{\top}\bm{e}_{g}\bm{e}_{g}^{\top}\bm{B}_{g}(\bm{U})\\\
&+\bm{B}_{g}(\bm{U})^{\top}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}\tilde{\bm{C}}^{\top}\bm{B}_{g}(\bm{U})+\\{\bm{B}_{g}(\bm{U})^{\top}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}\tilde{\bm{C}}^{\top}\bm{B}_{g}(\bm{U})\\}^{\top}.\end{aligned}$
(S8.15)

First, since $\bm{A}_{g}(\bm{U})\preceq\bm{W}_{g}$,

$\displaystyle\\{\bm{B}_{g}(\bm{U})^{\top}\bm{e}_{g}\bm{e}_{g}^{\top}\bm{B}_{g}(\bm{U})\\}\otimes\bm{A}_{g}(\bm{U})\preceq\\{\bm{B}_{g}(\bm{U})^{\top}\bm{e}_{g}\bm{e}_{g}^{\top}\bm{B}_{g}(\bm{U})\\}\otimes\bm{W}_{g},$

where

$\displaystyle\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\\{\bm{B}_{g}(\bm{U})^{\top}\bm{e}_{g}\bm{e}_{g}^{\top}\bm{B}_{g}(\bm{U})\\}\otimes\bm{W}_{g}\rVert_{2}\leq\max_{i\in[n]}(\lambda
p)^{-1}\sum_{g=1}^{p}w_{gi}\operatorname{Tr}\\{\bm{B}_{g}(\bm{U})^{\top}\bm{e}_{g}\bm{e}_{g}^{\top}\bm{B}_{g}(\bm{U})\\}.$

Identical techniques to those used to prove Lemma S8.12 can be used to show
that

$\displaystyle\sup_{\bm{U}\in\Omega_{\delta}}\max_{i\in[n]}(\lambda
p)^{-1}\sum_{g=1}^{p}w_{gi}\operatorname{Tr}\\{\bm{B}_{g}(\bm{U})^{\top}\bm{e}_{g}\bm{e}_{g}^{\top}\bm{B}_{g}(\bm{U})\\}=O_{P}(\lambda^{-1+\epsilon/2}\max_{(g,i)\in[p]\times[n]}w_{gi})$
$\displaystyle=O_{P}(\lambda^{-1+\epsilon}).$

Next, it is straightforward to show that for any $\epsilon\in(0,\eta)$,

$\displaystyle\sup_{\bm{U}\in\Omega_{\delta}}\max_{g\in[p]}\lVert\bm{B}_{g}(\bm{U})^{\top}\tilde{\bm{C}}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\tilde{\bm{C}}^{\top}\bm{B}_{g}(\bm{U})-\bm{v}_{u}^{-1}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\bm{v}_{u}^{-\top}\rVert_{2}=O_{P}(n^{-\eta+\epsilon}\lambda).$

Since $\lVert\bm{A}_{g}(\bm{U})\rVert_{2}\leq\lVert\bm{W}_{g}\rVert_{2}$, this
implies

$\displaystyle\sup_{\bm{U}\in\hat{\Omega}}\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\\{\bm{B}_{g}(\bm{U})^{\top}\tilde{\bm{C}}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\tilde{\bm{C}}^{\top}\bm{B}_{g}(\bm{U})\\}\otimes\bm{A}_{g}(U)-(\lambda
p)^{-1}\sum_{g=1}^{p}(\bm{v}_{u}^{-1}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\bm{v}_{u}^{-\top})\otimes\bm{A}_{g}(U)\rVert_{2}$
$\displaystyle=$ $\displaystyle O_{P}(n^{-\eta+\epsilon}),$

where

$\displaystyle(\lambda
p)^{-1}\sum_{g=1}^{p}(\bm{v}_{u}^{-1}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\bm{v}_{u}^{-\top})\otimes\bm{A}_{g}(U)=(\bm{v}_{u}^{-1}\otimes
I_{n})(\lambda
p)^{-1}\sum_{g=1}^{p}(\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top})\otimes\bm{A}_{g}(U)(\bm{v}_{u}^{-\top}\otimes
I_{n}),$

and for
$\tilde{\bm{s}}_{g}=\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}$,

$\displaystyle(\lambda
p)^{-1}\sum_{g=1}^{p}\tilde{\bm{s}}_{g}\otimes\bm{A}_{g}(U)=$
$\displaystyle(\lambda
p)^{-1}\sum_{g=1}^{p}\tilde{\bm{s}}_{g}\otimes\bm{P}_{g}^{\perp}-(\lambda
p)^{-1}\sum_{g=1}^{p}\tilde{\bm{s}}_{g}\otimes\\{\bm{P}_{g}^{\perp}\bm{U}(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\bm{U}^{\top}\bm{P}_{g}^{\perp}\\}$
$\displaystyle=$ $\displaystyle(\lambda
p)^{-1}\sum_{g=1}^{p}\tilde{\bm{s}}_{g}\otimes\bm{W}_{g}-(\lambda
p)^{-1}\sum_{g=1}^{p}\tilde{\bm{s}}_{g}\otimes\\{\bm{W}_{g}\bm{X}(\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}\bm{X}^{\top}\bm{W}_{g}\\}$
$\displaystyle-$ $\displaystyle(\lambda
p)^{-1}\sum_{g=1}^{p}\tilde{\bm{s}}_{g}\otimes\\{\bm{P}_{g}^{\perp}\bm{U}(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\bm{U}^{\top}\bm{P}_{g}^{\perp}\\}.$

Since $\delta=O_{P}(n^{-\eta})$ and by Lemma S8.9,

$\displaystyle\sup_{\bm{U}\in\Omega_{\delta}}\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\tilde{\bm{s}}_{g}\otimes\\{\bm{P}_{g}^{\perp}\bm{U}(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\bm{U}^{\top}\bm{P}_{g}^{\perp}\\}-(\lambda
p)^{-1}\sum_{g=1}^{p}\tilde{\bm{s}}_{g}\otimes\\{\bm{P}_{g}^{\perp}\tilde{\bm{C}}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\\}\rVert_{2}$
$\displaystyle=$ $\displaystyle O_{P}(n^{-\eta+\epsilon})$

for any $\epsilon\in(0,\eta)$. Next, for
$\bm{R}=n^{-1}\bm{C}^{\top}P_{X}^{\perp}\bm{C}$ and
$\bm{\Lambda}=np^{-1}\bm{L}^{\top}\bm{L}$,

$\displaystyle(\lambda
p)^{-1}\sum_{g=1}^{p}\tilde{\bm{s}}_{g}\otimes\bm{W}_{g}=\bm{R}^{1/2}\otimes
I_{n}\\{p^{-1}\sum_{g=1}^{p}(\lambda^{-1}\bm{\ell}_{g}\bm{\ell}_{g}^{\top})\otimes\bm{W}_{g}\\}\bm{R}^{1/2}\otimes
I_{n}$

such that $\lVert
p^{-1}\sum_{g=1}^{p}(\lambda^{-1}\bm{\ell}_{g_{r}}\bm{\ell}_{g_{s}})(\bm{W}_{g}-I_{n})\rVert_{2}=O_{P}(n^{-1/2+\epsilon})$
by Corollary S8.1, which implies

$\displaystyle\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\tilde{\bm{s}}_{g}\otimes\bm{W}_{g}-(\lambda^{-1}\bm{\Lambda})\otimes
I_{n}\rVert_{2}=O_{P}(n^{-1/2+\epsilon}).$

We also have that since $\max_{g\in[p]}\lVert
n^{-1/2}\bm{X}^{\top}\bm{W}_{g}\tilde{\bm{C}}\rVert_{2}=O_{P}(n^{-1/2+\epsilon})$
and $\max_{g\in[p]}\lVert
n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X}-n^{-1}\bm{X}^{\top}\bm{X}\rVert_{2}=O_{P}(n^{-1/2+\epsilon})$,

$\displaystyle\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{s}_{g}\otimes(\bm{W}_{g}\tilde{\bm{C}}\tilde{\bm{C}}^{\top}\bm{W}_{g})-(\lambda
p)^{-1}\sum_{g=1}^{p}\tilde{\bm{s}}_{g}\otimes(\bm{P}_{g}^{\perp}\tilde{\bm{C}}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp})\rVert_{2}=O_{P}(n^{-1/2+\epsilon}),\quad\bm{s}_{g}=\bm{\ell}_{g}\bm{\ell}_{g}^{\top}.$

For any $r,s\in[K]$, define $\bm{M}^{(rs)}=(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{s}_{g_{rs}}(\bm{W}_{g}\tilde{\bm{C}}\tilde{\bm{C}}^{\top}\bm{W}_{g})$.
Then

$\displaystyle\bm{M}^{(rs)}_{ij}=$
$\displaystyle\tilde{\bm{C}}_{i\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}}^{\top}\tilde{\bm{C}}_{j\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}}p^{-1}\sum_{g=1}^{p}(\bm{s}_{g_{rs}}/\lambda)(w_{gi}-1)(w_{gj}-1)+\tilde{\bm{C}}_{i\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}}^{\top}\tilde{\bm{C}}_{j\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}}p^{-1}\sum_{g=1}^{p}(\bm{s}_{g_{rs}}/\lambda)(w_{gi}-1)$
$\displaystyle+\tilde{\bm{C}}_{i\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}}^{\top}\tilde{\bm{C}}_{j\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}}p^{-1}\sum_{g=1}^{p}(\bm{s}_{g_{rs}}/\lambda)(w_{gj}-1)+\tilde{\bm{C}}_{i\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}}^{\top}\tilde{\bm{C}}_{j\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}}p^{-1}\sum_{g=1}^{p}(\bm{s}_{g_{rs}}/\lambda),\quad
i,j\in[n].$

Therefore,

$\displaystyle\bm{M}^{(rs)}$
$\displaystyle=\sum_{k=1}^{K}\operatorname{diag}(\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k})\\{p^{-1}\bm{W}^{\top}\bm{S}^{(rs)}\bm{W}\\}\operatorname{diag}(\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k})+\bar{\bm{C}}^{(rs)}\tilde{\bm{C}}+\\{\bar{\bm{C}}^{(rs)}\tilde{\bm{C}}\\}^{\top}+\bm{\Lambda}_{rs}\tilde{\bm{C}}\tilde{\bm{C}}^{\top}$
$\displaystyle\bm{W}_{gi}$
$\displaystyle=w_{gi}-1,\quad\bar{\bm{C}}^{(rs)}_{i\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}}=p^{-1}\sum_{g=1}^{p}(\bm{s}_{g_{rs}}/\lambda)(w_{gi}-1)\tilde{\bm{C}}_{i\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}},\quad
g\in[p];i\in[n]$ $\displaystyle\bm{S}^{(rs)}$
$\displaystyle=\operatorname{diag}(\bm{s}_{1_{rs}}/\lambda,\ldots,\bm{s}_{p_{rs}}/\lambda).$

First, for some constant $c>0$,

$\displaystyle\lVert\operatorname{diag}(\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k})\\{p^{-1}\bm{W}^{\top}\bm{S}^{(rs)}\bm{W}\\}\operatorname{diag}(\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k})\rVert_{2}\leq
c\lVert
p^{-1/2}\bm{W}\rVert_{2}^{2}\max_{i\in[n]}\tilde{\bm{C}}_{ik}^{2}=O_{P}(n^{-1+\epsilon})$

by Lemma S8.6. Next, it is easy to see that
$\lVert\bar{\bm{C}}^{(rs)}\tilde{\bm{C}}\rVert_{2}\leq\lVert\bar{\bm{C}}^{(rs)}\rVert_{2}=O_{P}(n^{-1/2+\epsilon})$.
Lastly, since the entries of $\bm{X}$ are uniformly bounded, identical
techniques can be used to show that

$\displaystyle\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\tilde{\bm{s}}_{g}\otimes\\{\bm{W}_{g}\bm{X}(\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}\bm{X}^{\top}\bm{W}_{g}\\}-\bm{\Lambda}\otimes
P_{X}\rVert_{2}=O_{P}(n^{-1/2+\epsilon}).$

Putting this all together implies

$\displaystyle\sup_{\bm{U}\in\Omega_{\delta}}\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\\{\bm{B}_{g}(\bm{U})^{\top}\tilde{\bm{C}}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\tilde{\bm{C}}^{\top}\bm{B}_{g}(\bm{U})\\}\otimes\bm{A}_{g}(\bm{U})-(\lambda^{-1}\bm{v}_{u}^{-1}\bm{\Lambda}\bm{v}_{u}^{-\top})\otimes\bm{Q}\bm{Q}^{\top}\rVert_{2}=O_{P}(n^{-\eta+\epsilon}),$

where $\lVert\bm{v}_{u}-\bm{v}\rVert_{2}=O(\delta^{2})=O_{P}(n^{-2\eta})$ for
some unitary matrix $\bm{v}\in\mathbb{R}^{K\times K}$ by Lemma S8.3.

For the remaining two terms in (S8.15), we note that for

$\displaystyle\bm{S}(\bm{U})=[\bm{B}_{1}(\bm{U})^{\top}\tilde{\bm{C}}\tilde{\bm{\ell}}_{1}\cdots\bm{B}_{p}(\bm{U})^{\top}\tilde{\bm{C}}\tilde{\bm{\ell}}_{p}],\,\bm{T}(\bm{U})=[\bm{B}_{1}(\bm{U})^{\top}\bm{e}_{1}\cdots\bm{B}_{p}(\bm{U})^{\top}\bm{e}_{p}]\in\mathbb{R}^{K\times
p},$ $\displaystyle\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\\{\bm{B}_{g}(\bm{U})^{\top}\tilde{\bm{C}}\tilde{\bm{\ell}}_{g}\bm{e}_{g}^{\top}\bm{B}_{g}(\bm{U})\\}\otimes\bm{A}_{g}(\bm{U})\rVert_{2}\leq$
$\displaystyle\lVert(\lambda
p)^{-1}\bm{S}(\bm{U})\bm{S}(\bm{U})^{\top}\rVert_{2}^{1/2}$
$\displaystyle\times\lVert(\lambda
p)^{-1}\bm{T}(\bm{U})\bm{T}(\bm{U})^{\top}\rVert_{2}^{1/2}.$

Our above work shows that $\lVert(\lambda
p)^{-1}\bm{S}(\bm{U})\bm{S}(\bm{U})^{\top}\rVert_{2}^{1/2}=O_{P}(1)$ and
$\lVert(\lambda
p)^{-1}\bm{T}(\bm{U})\bm{T}(\bm{U})^{\top}\rVert_{2}^{1/2}=O_{P}(n^{-\eta+\epsilon}+\lambda^{-1/2+\epsilon})$,
which completes the proof. ∎

######  Lemma S8.16 (Second and fourth terms of (S8.13)).

Suppose the assumptions of Lemma S8.14 hold, let $\eta\in(0,1/2)$, and let
$\Omega_{\delta}$ be as defined in Lemma S8.13. Then if
$\delta=O_{P}(n^{-\eta})$,

$\displaystyle\sup_{\bm{U}\in\Omega_{\delta}}\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{B}_{g}(\bm{U})^{\top}\otimes\\{\bm{A}_{g}(\bm{U})\bm{y}_{g}\bm{y}_{g}^{\top}\bm{B}_{g}(\bm{U})\\}\bm{\Pi}\rVert_{2}=O_{P}(n^{-\eta+\epsilon}+\lambda^{-1/2+\epsilon})$

for any constant $\epsilon>0$.

###### Proof.

Since $\bm{\Pi}$ is a permutation matrix,

$\displaystyle\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{B}_{g}(\bm{U})^{\top}\otimes\\{\bm{A}_{g}(\bm{U})\bm{y}_{g}\bm{y}_{g}^{\top}\bm{B}_{g}(\bm{U})\\}\bm{\Pi}\rVert_{2}\leq\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{B}_{g}(\bm{U})^{\top}\otimes\\{\bm{A}_{g}(\bm{U})\bm{y}_{g}\bm{y}_{g}^{\top}\bm{B}_{g}(\bm{U})\\}\rVert_{2}.$

By the definition of $\bm{B}_{g}(\bm{U})$,

$\displaystyle\bm{B}_{g}(\bm{U})^{\top}\otimes\\{\bm{A}_{g}(\bm{U})\bm{y}_{g}\bm{y}_{g}^{\top}\bm{B}_{g}(\bm{U})\\}$
$\displaystyle=$
$\displaystyle[(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1/2}\otimes\\{\bm{A}_{g}(\bm{U})\bm{y}_{g}\\}][\\{\bm{P}_{g}^{\perp}\bm{U}(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1/2}\\}\otimes\\{\bm{B}_{g}(\bm{U})^{\top}\bm{y}_{g}\\}]^{\top}.$

Define

$\displaystyle\bm{S}(\bm{U})$
$\displaystyle=\begin{pmatrix}(\bm{U}^{\top}\bm{P}_{1}^{\perp}\bm{U})^{-1/2}\otimes\\{\bm{A}_{1}(\bm{U})\bm{y}_{1}\\}&\cdots&(\bm{U}^{\top}\bm{P}_{p}^{\perp}\bm{U})^{-1/2}\otimes\\{\bm{A}_{p}(\bm{U})\bm{y}_{p}\\}\end{pmatrix}$
$\displaystyle\bm{T}(\bm{U})$
$\displaystyle=\begin{pmatrix}\\{\bm{P}_{1}^{\perp}\bm{U}(\bm{U}^{\top}\bm{P}_{1}^{\perp}\bm{U})^{-1/2}\\}\otimes\\{\bm{B}_{1}(\bm{U})^{\top}\bm{y}_{1}\\}&\cdots&\\{\bm{P}_{p}^{\perp}\bm{U}(\bm{U}^{\top}\bm{P}_{p}^{\perp}\bm{U})^{-1/2}\\}\otimes\\{\bm{B}_{p}(\bm{U})^{\top}\bm{y}_{p}\\}\end{pmatrix},$

where $\bm{S}(\bm{U}),\bm{T}(\bm{U})\in\mathbb{R}^{nK\times pK}$ and
$\sum_{g=1}^{p}\bm{B}_{g}(\bm{U})^{\top}\otimes\\{\bm{A}_{g}(\bm{U})\bm{y}_{g}\bm{y}_{g}^{\top}\bm{B}_{g}(\bm{U})\\}=\bm{S}(\bm{U})\\{\bm{T}(\bm{U})\\}^{\top}$.
Therefore,

$\displaystyle\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{B}_{g}(\bm{U})^{\top}\otimes\\{\bm{A}_{g}(\bm{U})\bm{y}_{g}\bm{y}_{g}^{\top}\bm{B}_{g}(\bm{U})\\}\rVert_{2}$
$\displaystyle\leq$ $\displaystyle\lVert(\lambda
p)^{-1}\bm{S}(\bm{U})\\{\bm{S}(\bm{U})\\}^{\top}\rVert_{2}^{1/2}\lVert(\lambda
p)^{-1}\bm{T}(\bm{U})\\{\bm{T}(\bm{U})\\}^{\top}\rVert_{2}^{1/2},$

where $\sup_{\bm{U}\in\Omega_{\delta}}\lVert(\lambda
p)^{-1}\bm{S}(\bm{U})\\{\bm{S}(\bm{U})\\}^{\top}\rVert_{2}=O_{P}(n^{-2\eta+\epsilon}+\lambda^{-1+\epsilon})$
by Lemma S8.14. We also see that

$\displaystyle(\lambda
p)^{-1}\bm{T}(\bm{U})\\{\bm{T}(\bm{U})\\}^{\top}=(\lambda
p)^{-1}\sum_{g=1}^{p}\\{\bm{P}_{g}^{\perp}\bm{U}(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-\top}\bm{U}^{\top}\bm{P}_{g}^{\perp}\\}\otimes\\{\bm{B}_{g}(\bm{U})^{\top}\bm{y}_{g}\bm{y}_{g}^{\top}\bm{B}_{g}(\bm{U})\\},$

where the same techniques used to prove Lemma S8.15 can be used to show that

$\displaystyle\sup_{\bm{U}\in\Omega_{\delta}}\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\\{\bm{P}_{g}^{\perp}\bm{U}(\bm{U}^{\top}\bm{P}_{g}^{\perp}\bm{U})^{-1}\bm{U}^{\top}\bm{P}_{g}^{\perp}\\}\otimes\\{\bm{B}_{g}(\bm{U})^{\top}\bm{y}_{g}\bm{y}_{g}^{\top}\bm{B}_{g}(\bm{U})\\}\rVert_{2}=O_{P}(1),$

which completes the proof. ∎

######  Corollary S8.3.

Let $\tilde{\bm{H}}(\bm{U})$ be as defined in (S8.13),
$\bm{\Lambda}=np^{-1}\bm{L}^{\top}\bm{L}$, let $\eta\in(0,1/2)$, and let
$\Omega_{\delta}$ be as defined in Lemma S8.13. Then if the assumptions of
Lemma S8.14 hold and $\delta=O_{P}(n^{-\eta})$, there exists a unitary matrix
$\bm{v}=\bm{v}(\bm{U})\in\mathbb{R}^{K\times K}$ for each
$\bm{U}\in\Omega_{\delta}$ such that
$\sup_{\bm{U}\in\Omega_{\delta}}\lVert\tilde{\bm{H}}(\bm{U})+(\lambda^{-1}\bm{v}^{\top}\bm{\Lambda}\bm{v})\otimes(\bm{0}_{K\times
K}\oplus
I_{n-d-K})\rVert_{2}=O_{P}(n^{-\eta+\epsilon}+\lambda^{-1/2+\epsilon})$ for
any constant $\epsilon>0$.

###### Proof.

This follows directly from Lemmas S8.14, S8.15, and S8.16. ∎

######  Remark S8.9.

We can construct $\bm{v}=\bm{v}(\bm{U})$ using the following procedure. For
$\bm{U}\in\Omega_{\delta}$ , let $\bm{v}_{u}$ be as defined in Lemma S8.3, and
let $\bm{v}_{u}=\bm{A}_{u}\bm{\Sigma}_{u}\bm{B}_{u}^{\top}$ be its singular
value decomposition. By the proof of Lemma S8.15, Corollary S8.3 holds with
$\bm{v}$ replaced with $\bm{v}_{u}^{-\top}$. Since $\lVert
I_{K}-\bm{\Sigma}_{u}\rVert_{2}=O(\delta^{2})$ by the proof of Lemma S8.3,
Corollary S8.3 holds with $\bm{v}=\bm{A}_{u}\bm{B}_{u}^{\top}$.

######  Lemma S8.17 (First term in (S8.12)).

Under the assumptions of Lemma S8.14 and for any $\epsilon\in(0,1/2)$,

$\displaystyle\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{P}_{g}^{\perp}\tilde{\bm{C}}(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}})^{-1}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}\rVert_{2}=O_{P}(\lambda^{-1+\epsilon}).$

###### Proof.

Without loss of generality, we may assume $n^{-1}\bm{X}^{\top}\bm{X}=I_{d}$.
Then

$\displaystyle\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}=\tilde{\bm{C}}^{\top}\bm{W}_{g}\bm{e}_{g}-\tilde{\bm{C}}^{\top}\bm{W}_{g}\bm{X}(\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{e}_{g}.$

For $\bm{R}=n^{-1}\bm{C}^{\top}P_{X}^{\perp}\bm{C}$,

$\displaystyle\tilde{\bm{C}}^{\top}\bm{W}_{g}\bm{e}_{g}=\bm{R}^{-1/2}\\{n^{-1/2}\bm{C}^{\top}\bm{e}_{g}+n^{-1/2}\bm{C}^{\top}(\bm{W}_{g}-I_{n})\bm{e}_{g}\\}-\bm{R}^{-1/2}(n^{-1}\bm{C}^{\top}\bm{X})(n^{-1/2}\bm{X}^{\top}\bm{W}_{g}\bm{e}_{g}),$

where $\max_{g\in[p]}\lVert
n^{-1/2}\bm{C}^{\top}\bm{e}_{g}\rVert_{2}=O_{P}(n^{\epsilon})$ by Lemma S8.2
and $\max_{g\in[p]}\lVert
n^{-1/2}\bm{C}^{\top}(\bm{W}_{g}-I_{n})\bm{e}_{g}\rVert_{2}=O_{P}(n^{\epsilon})$
by Lemma S8.4. An identical argument can be used to show that
$\max_{g\in[p]}\lVert
n^{-1/2}\bm{X}^{\top}\allowbreak\bm{W}_{g}\allowbreak\bm{e}_{g}\rVert_{2}=O_{P}(n^{\epsilon})$,
which implies
$\max_{g\in[p]}\lVert\tilde{\bm{C}}^{\top}\bm{W}_{g}\bm{e}_{g}\rVert_{2}=O_{P}(n^{\epsilon})$.
A similar argument can be used to show that

$\displaystyle\max_{g\in[p]}\lVert\tilde{\bm{C}}^{\top}\bm{W}_{g}\bm{X}(\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{e}_{g}\rVert_{2}=O_{P}(n^{-1/2+\epsilon}).$

Putting all this together implies that
$\max_{g\in[p]}\lVert\tilde{\bm{C}}^{\top}\bm{W}_{g}\bm{e}_{g}\rVert_{2}=O_{P}(n^{\epsilon})$,
which by (S8.7) in Lemma S8.9 and the fact that
$\max_{g\in[p]}\lVert\tilde{\bm{C}}^{\top}(\bm{P}_{g}^{\perp})^{2}\tilde{\bm{C}}\rVert_{2}\leq
c\\{1+o_{P}(1)\\}$, further implies

$\displaystyle\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{P}_{g}^{\perp}\tilde{\bm{C}}(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}})^{-1}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}-(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}\rVert_{2}$
$\displaystyle=O_{P}(n^{-1/2+\epsilon}\lambda^{-1/2})$
$\displaystyle=O_{P}(\lambda^{-1+\epsilon}).$

Next,

$\displaystyle\begin{aligned}
\bm{P}_{g}^{\perp}\tilde{\bm{C}}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}=&\bm{W}_{g}\tilde{\bm{C}}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}\\\
&-(n^{-1/2}\bm{W}_{g}\bm{X})(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}(n^{-1/2}\bm{X}^{\top}\bm{W}_{g}\tilde{\bm{C}})\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}.\end{aligned}$
(S8.16)

Starting with the second term in (S8.16),

$\displaystyle\lVert(n^{-1/2}\bm{W}_{g}\bm{X})(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}(n^{-1/2}\bm{X}^{\top}\bm{W}_{g}\tilde{\bm{C}})\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}\rVert_{2}$
$\displaystyle\leq$ $\displaystyle\lVert
n^{-1/2}\bm{W}_{g}\bm{X}\rVert_{2}\lVert(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}\rVert_{2}\lVert
n^{-1/2}\bm{X}^{\top}\bm{W}_{g}\tilde{\bm{C}}\rVert_{2}\lVert\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\rVert_{2}\lVert\tilde{\bm{\ell}}_{g}\rVert_{2},$

where Lemma S8.4 and the above derivation of the behavior of
$\lVert\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\rVert_{2}$ implies
that for some constant $c>0$,

$\displaystyle\max_{g\in[p]}\lVert
n^{-1/2}\bm{W}_{g}\bm{X}\rVert_{2},\,\max_{g\in[p]}\lVert(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}\rVert_{2}\leq
c\\{1+o_{P}(1)\\}$ $\displaystyle\max_{g\in[p]}\lVert
n^{-1/2}\bm{X}^{\top}\bm{W}_{g}\tilde{\bm{C}}\rVert_{2}=O_{P}(n^{-1/2+\epsilon}),\quad\max_{g\in[p]}\lVert\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\rVert_{2}=O_{P}(n^{\epsilon}).$

Therefore,

$\displaystyle\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}(n^{-1/2}\bm{W}_{g}\bm{X})(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}(n^{-1/2}\bm{X}^{\top}\bm{W}_{g}\tilde{\bm{C}})\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}\rVert_{2}=O_{P}(\lambda^{-1+\epsilon}).$

The first term in (S8.16) can be expressed as

$\displaystyle\bm{W}_{g}\tilde{\bm{C}}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}=$
$\displaystyle\bm{W}_{g}\tilde{\bm{C}}\tilde{\bm{C}}^{\top}\bm{W}_{g}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}$
$\displaystyle-\bm{W}_{g}\tilde{\bm{C}}(n^{-1/2}\tilde{\bm{C}}^{\top}\bm{W}_{g}\bm{X})(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}(n^{-1/2}\bm{X}^{\top}\bm{W}_{g}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top})$
$\displaystyle=$
$\displaystyle\sum_{k=1}^{K}\bm{W}_{g}\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}^{\top}\bm{W}_{g}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}$
$\displaystyle-\bm{W}_{g}\tilde{\bm{C}}(n^{-1/2}\tilde{\bm{C}}^{\top}\bm{W}_{g}\bm{X})(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}(n^{-1/2}\bm{X}^{\top}\bm{W}_{g}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top})$

where an identical analysis to the one above can be used to show that

$\displaystyle\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{W}_{g}\tilde{\bm{C}}(n^{-1/2}\tilde{\bm{C}}^{\top}\bm{W}_{g}\bm{X})(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}(n^{-1/2}\bm{X}^{\top}\bm{W}_{g}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top})\rVert_{2}=O_{P}(\lambda^{-1+\epsilon}).$

Lastly,

$\displaystyle\begin{aligned} (\lambda
p)^{-1}\sum_{g=1}^{p}\bm{W}_{g}\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}^{\top}\bm{W}_{g}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}=&(\lambda
p)^{-1}\sum_{g=1}^{p}(\bm{W}_{g}-I_{n})\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}^{\top}(\bm{W}_{g}-I_{n})\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}\\\
&+(\lambda
p)^{-1}\sum_{g=1}^{p}(\bm{W}_{g}-I_{n})\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}^{\top}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}\\\
&+\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}(\lambda
p)^{-1}\sum_{g=1}^{p}\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}^{\top}(\bm{W}_{g}-I_{n})\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}\\\
&-\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}(\lambda
p)^{-1}\sum_{g=1}^{p}\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}^{\top}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}.\end{aligned}$
(S8.17)

Result (S8.3c) in Lemma S8.2 and Remark S8.7 imply
$\lVert\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}(\lambda
p)^{-1}\sum_{g=1}^{p}\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}^{\top}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}\rVert_{2}\allowbreak=\allowbreak
O_{P}\\{\allowbreak(\lambda p)^{-1/2}\\}$. For the third term in (S8.17), we
see that

$\displaystyle(\lambda
p)^{-1}\sum_{g=1}^{p}\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}^{\top}(\bm{W}_{g}-I_{n})\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}=(\bm{R}^{-1/2})_{k\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}}^{\top}(\lambda
p)^{-1}\sum_{g=1}^{p}n^{-1/2}\bm{C}^{\top}(\bm{W}_{g}-I_{n})\bm{e}_{g}(n^{1/2}\bm{\ell}_{g})^{\top}\bm{R}^{1/2}$
$\displaystyle-(\bm{R}^{-1/2})_{k\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}}^{\top}(n^{-1}\bm{C}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}^{\top}\bm{X})(\lambda
p)^{-1}\sum_{g=1}^{p}(n^{-1/2}\bm{X})^{\top}(\bm{W}_{g}-I_{n})\bm{e}_{g}(n^{1/2}\bm{\ell}_{g})^{\top}\bm{R}^{1/2}.$

Since $\operatorname{\mathbb{V}}\\{(\bm{W}_{g}-I_{n})\bm{e}_{g}\\}$ is a
diagonal matrix with uniformly bounded diagonal entries,

$\displaystyle\lVert\bm{R}^{-1/2}(n^{-1}\bm{C}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}^{\top}\bm{X})(\lambda
p)^{-1}\sum_{g=1}^{p}(n^{-1/2}\bm{X})^{\top}(\bm{W}_{g}-I_{n})\bm{e}_{g}(n^{1/2}\bm{\ell}_{g})^{\top}\rVert_{2}=O_{P}\\{(\lambda
p)^{-1/2}\\}.$

Next, for $r,s\in[K]$ and some constants $c_{1},c_{2}>0$,

$\displaystyle\operatorname{\mathbb{V}}\\{(\lambda
p)^{-1}\sum_{g=1}^{p}n^{-1/2}\bm{C}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}r}^{\top}(\bm{W}_{g}-I_{n})\bm{e}_{g}(n^{1/2}\bm{\ell}_{g_{s}})\\}\leq$
$\displaystyle
c_{1}\lambda^{-1}p^{-2}\sum_{g=1}^{p}[n^{-1}\sum_{i=1}^{n}\operatorname{\mathbb{E}}\\{\bm{C}_{ir}^{2}(w_{gi}-1)^{2}\bm{e}_{g_{i}}^{2}\\}]$
$\displaystyle\leq$ $\displaystyle c_{1}c_{2}(\lambda p)^{-1},$

which implies the third term in (S8.17) is $O_{P}\\{(\lambda p)^{-1/2}\\}$.
The $i$th row of the second term in (S8.17) can be expressed as

$\displaystyle\tilde{\bm{C}}_{ik}\bm{R}^{1/2}\left\\{(\lambda
p)^{-1}\sum_{g=1}^{p}(n^{1/2}\bm{\ell}_{g})(w_{gi}-1)\bm{e}_{g}^{\top}(n^{-1/2}\bm{C})\right\\}(\bm{R}^{-1/2})_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}$
$\displaystyle-$ $\displaystyle\tilde{\bm{C}}_{ik}\bm{R}^{1/2}\left\\{(\lambda
p)^{-1}\sum_{g=1}^{p}(n^{1/2}\bm{\ell}_{g})(w_{gi}-1)\bm{e}_{g}^{\top}(n^{-1/2}\bm{X})\right\\}(n^{-1}\bm{X}^{\top}\bm{C})(\bm{R}^{-1/2})_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}\in\mathbb{R}^{K},$

where
$\operatorname{\mathbb{E}}\\{\bm{e}_{g}^{\top}(n^{-1/2}\bm{C}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}r})^{(2m)}\\}\leq
c_{m}$ for some constant $c_{m}>0$ that only depends on the integer $m>0$ by
Lemma S8.2. As a consequence, Corollary S8.1 implies

$\displaystyle\max_{i\in[n]}\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}(n^{1/2}\bm{\ell}_{g})(w_{gi}-1)\bm{e}_{g}^{\top}(n^{-1/2}\bm{C})\rVert_{2}=O_{P}(\lambda^{-1/2}p^{-1/2+\epsilon})=O_{P}(\lambda^{-1+\epsilon})$
$\displaystyle\max_{i\in[n]}\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}(n^{1/2}\bm{\ell}_{g})(w_{gi}-1)\bm{e}_{g}^{\top}(n^{-1/2}\bm{X})\rVert_{2}=O_{P}(\lambda^{-1/2}p^{-1/2+\epsilon})=O_{P}(\lambda^{-1+\epsilon}),$

which because $\sum_{i=1}^{n}\tilde{\bm{C}}_{ik}^{2}=1$, proves the second
term in (S8.17) is $O_{P}(\lambda^{-1+\epsilon})$. We can then express the
$i$th row of first term in (S8.17) as

$\displaystyle\begin{aligned} &\tilde{\bm{C}}_{ik}^{2}\bm{R}^{1/2}(\lambda
p)^{-1}\sum_{g=1}^{p}(n^{1/2}\bm{\ell}_{g})(w_{gi}-1)^{2}\bm{e}_{g_{i}}\\\
+&\tilde{\bm{C}}_{ik}\bm{R}^{1/2}(\lambda
p)^{-1}\sum_{g=1}^{p}(n^{1/2}\bm{\ell}_{g})(w_{gi}-1)\sum_{j\neq
i}^{n}\tilde{\bm{C}}_{jk}\bm{e}_{g_{j}}(w_{gj}-1).\end{aligned}$ (S8.18)

First,

$\displaystyle\max_{i\in[n]}\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}(n^{1/2}\bm{\ell}_{g})(w_{gi}-1)^{2}\bm{e}_{g_{i}}\rVert_{2}=O_{P}(\lambda^{-1/2})$

and

$\displaystyle\tilde{\bm{C}}_{ik}=n^{-1/2}\bm{C}_{i\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}}^{\top}(\bm{R}^{-1/2})_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}-n^{-1/2}\bm{X}_{i\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}}(n^{-1}\bm{X}^{\top}\bm{C})(\bm{R}^{-1/2})_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k},$

which implies
$\max_{i\in[\tilde{\bm{C}}_{ik}]}\tilde{\bm{C}}_{ik}^{2}=O_{P}(n^{-1+\epsilon})$,
and consequently that

$\displaystyle\lVert\tilde{\bm{C}}_{ik}^{2}\bm{R}^{1/2}(\lambda
p)^{-1}\sum_{g=1}^{p}(n^{1/2}\bm{\ell}_{g})(w_{gi}-1)^{2}\bm{e}_{g_{i}}\rVert_{2}\leq\lambda^{-1/2}O_{P}\\{(\max_{i\in[n]}n\tilde{\bm{C}}_{ik}^{4})^{1/2}\\}=O_{P}(\lambda^{-1+\epsilon}).$

Finally, the second term in (S8.18) can be expressed as

$\displaystyle\tilde{\bm{C}}_{ik}\bm{R}^{1/2}\left\\{(\lambda
p)^{-1}\sum_{g=1}^{p}(n^{1/2}\bm{\ell}_{g})(w_{gi}-1)\sum_{j\neq
i}^{n}\bm{e}_{g_{j}}(w_{gj}-1)(n^{-1/2}\bm{C}_{j\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}})^{\top}\right\\}(\bm{R}^{-1/2})_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}$
$\displaystyle-$ $\displaystyle\tilde{\bm{C}}_{ik}\bm{R}^{1/2}\left\\{(\lambda
p)^{-1}\sum_{g=1}^{p}(n^{1/2}\bm{\ell}_{g})(w_{gi}-1)\sum_{j\neq
i}^{n}\bm{e}_{g_{j}}(w_{gj}-1)(n^{-1/2}\bm{X}_{j\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}})^{\top}\right\\}(n^{-1}\bm{X}^{\top}\bm{C})(\bm{R}^{-1/2})_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k}.$

Since

$\displaystyle\operatorname{\mathbb{E}}\left[\left\\{\sum_{j\neq
i}^{n}\bm{e}_{g_{j}}(w_{gj}-1)(n^{-1/2}\bm{C}_{jr})\right\\}^{2m}\right],\,\operatorname{\mathbb{E}}\left[\left\\{\sum_{j\neq
i}^{n}\bm{e}_{g_{j}}(w_{gj}-1)(n^{-1/2}\bm{X}_{js})\right\\}^{2m}\right]\leq
c_{m}$

for $r\in[K]$, $s\in[d]$, and any integer $m>0$ and constant $c_{m}$ that only
depends on $m$ by the proofs of Lemmas S8.2 and S8.4, Corollary S8.1 implies

$\displaystyle\max_{i\in[n]}\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}(n^{1/2}\bm{\ell}_{g})(w_{gi}-1)\sum_{j\neq
i}^{n}\bm{e}_{g_{j}}(w_{gj}-1)(n^{-1/2}\bm{C}_{j\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}})^{\top}\rVert_{2}=O_{P}(\lambda^{-1/2}p^{-1/2+\epsilon})=O_{P}(\lambda^{-1+\epsilon})$
$\displaystyle\max_{i\in[n]}\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}(n^{1/2}\bm{\ell}_{g})(w_{gi}-1)\sum_{j\neq
i}^{n}\bm{e}_{g_{j}}(w_{gj}-1)(n^{-1/2}\bm{X}_{j\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}})^{\top}\rVert_{2}=O_{P}(\lambda^{-1/2}p^{-1/2+\epsilon})=O_{P}(\lambda^{-1+\epsilon}).$

Since $\sum_{i=1}^{n}\tilde{\bm{C}}_{ik}^{2}=1$, this shows the first term in
(S8.17) is $O_{P}(\lambda^{-1+\epsilon})$, and completes the proof. ∎

######  Lemma S8.18 (First term in (S8.12)).

Suppose the assumptions of Lemma S8.14 hold and let $\bm{s}^{(1)}=(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{P}_{g}^{\perp}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}$.
Then for any constant $\epsilon\in(0,1/2)$,
$\lVert\bm{s}^{(1)}\rVert_{2}=O_{P}(\lambda^{-1/2+\epsilon})$.

###### Proof.

Without loss of generality, assume $n^{-1}\bm{X}^{\top}\bm{X}=I_{K}$. We can
express $\bm{P}_{g}^{\perp}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}$ as

$\displaystyle\bm{P}_{g}^{\perp}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}=\bm{W}_{g}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}-n^{-1/2}\bm{W}_{g}\bm{X}(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}(n^{-1/2}\bm{X}^{\top}\bm{W}_{g}\bm{e}_{g})\tilde{\bm{\ell}}_{g}^{\top}.$

The same techniques used to prove Lemma S8.17 can be used to show

$\displaystyle\max_{g\in[p]}\lVert
n^{-1/2}\bm{W}_{g}\bm{X}(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}(n^{-1/2}\bm{X}^{\top}\bm{W}_{g}\bm{e}_{g})\rVert_{2}=n^{\epsilon}$

for any $\epsilon\in(0,1/2)$, which implies

$\displaystyle\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}n^{-1/2}\bm{W}_{g}\bm{X}(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}(n^{-1/2}\bm{X}^{\top}\bm{W}_{g}\bm{e}_{g})\tilde{\bm{\ell}}_{g}^{\top}\rVert_{2}=O_{P}(\lambda^{-1/2+\epsilon}).$

Next,

$\displaystyle\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{W}_{g}\bm{e}_{g}\tilde{\bm{\ell}}_{g}^{\top}\rVert_{2}=\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{W}_{g}\bm{e}_{g}(n^{1/2}\bm{\ell}_{g})^{\top}\rVert_{2}O_{P}(1).$

To prove the results, we therefore only have to show that

$\displaystyle\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{W}_{g}\bm{e}_{g}(n^{1/2}\bm{\ell}_{g})^{\top}\rVert_{2}=O_{P}(\lambda^{-1/2+\epsilon}),$

which follows because for any $k\in[K]$ and some constants $c_{1},c_{2}>0$,

$\displaystyle\operatorname{\mathbb{E}}\left[\sum_{i=1}^{n}\\{(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{W}_{g}\bm{e}_{g}(n^{1/2}\bm{\ell}_{g_{k}})\\}_{i}^{2}\right]=$
$\displaystyle(\lambda
p)^{-2}\operatorname{Tr}\\{\operatorname{\mathbb{V}}(\sum_{g=1}^{p}\bm{W}_{g}\bm{e}_{g}n^{1/2}\bm{\ell}_{g_{k}})\\}$
$\displaystyle=$ $\displaystyle(\lambda
p)^{-2}\sum_{g=1}^{p}\operatorname{Tr}[\operatorname{\mathbb{E}}\\{\operatorname{\mathbb{V}}(\bm{W}_{g}\bm{e}_{g}n^{1/2}\bm{\ell}_{g_{k}}\mid\bm{C})\\}]$
$\displaystyle\leq$ $\displaystyle
c_{1}\lambda^{-1}p^{-2}\sum_{g=1}^{p}\operatorname{Tr}\\{\operatorname{\mathbb{V}}(\bm{W}_{g}\bm{e}_{g})\\}\leq
c_{1}c_{2}\lambda^{-1}.$

∎

######  Lemma S8.19 (Second term in (S8.12)).

Define

$\displaystyle\bm{s}^{(2)}=(p\lambda)^{-1}\sum_{g=1}^{p}\\{\bm{P}_{g}^{\perp}-\bm{P}_{g}^{\perp}\tilde{\bm{C}}(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}})^{-1}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\\}\bm{e}_{g}\bm{e}_{g}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}})^{-1}.$

Then under the assumptions of Lemma S8.14 and for any $\epsilon\in(0,1/2)$,
$\lVert\bm{s}^{(2)}\rVert_{2}=O_{P}(\lambda^{-1+\epsilon})$

###### Proof.

Define
$\bm{S}=\begin{pmatrix}\bm{A}_{1}(\tilde{\bm{C}})\bm{e}_{1}\cdots\bm{A}_{p}(\tilde{\bm{C}})\bm{e}_{p}\end{pmatrix}\in\mathbb{R}^{n\times
p}$ and
$\bm{T}=\begin{pmatrix}\bm{B}_{1}(\tilde{\bm{C}})^{\top}\bm{e}_{1}\cdots\bm{B}_{p}(\tilde{\bm{C}})^{\top}\bm{e}_{p}\end{pmatrix}\in\mathbb{R}^{K\times
p}$. Then $\bm{s}^{(2)}=(\lambda p)^{-1}\bm{S}\bm{T}^{\top}$, which implies

$\displaystyle\lVert\bm{s}^{(2)}\rVert_{2}\leq\lVert(\lambda
p)^{-1}\bm{S}\bm{S}^{\top}\rVert_{2}^{1/2}\lVert(\lambda
p)^{-1}\bm{T}\bm{T}^{\top}\rVert_{2}^{1/2}.$

The proof of Lemma S8.14 shows that

$\displaystyle\lVert(\lambda
p)^{-1}\bm{S}\bm{S}^{\top}\rVert_{2}=\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{A}_{g}(\tilde{\bm{C}})\bm{e}_{g}\bm{e}_{g}^{\top}\bm{A}_{g}(\tilde{\bm{C}})\rVert_{2}=O_{P}(\lambda^{-1+\epsilon})$

and the proof of Lemma S8.15 shows that

$\displaystyle\lVert(\lambda
p)^{-1}\bm{T}\bm{T}^{\top}\rVert_{2}=\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{B}_{g}(\tilde{\bm{C}})^{\top}\bm{e}_{g}\bm{e}_{g}^{\top}\bm{B}_{g}(\tilde{\bm{C}})\rVert_{2}=O_{P}(\lambda^{-1+\epsilon}).$

∎

######  Theorem S8.1.

Suppose the assumptions of Lemma S8.14 hold and let
$\bm{\Lambda}=np^{-1}\bm{L}^{\top}\bm{L}$, $f$ and $\Omega_{\delta}$ be as
defined in (S8.1) and Lemma S8.13, respectively, and
$\hat{\bm{C}}\in\operatorname*{argmax}_{\bm{U}\in\Omega_{\delta}}f(\bm{U})$.
Then there exist $\hat{\bm{v}}\in\mathbb{R}^{K\times K}$,
$\hat{\bm{z}}\in\mathbb{R}^{(n-d-K)\times K}$, and a unitary matrix
$\bm{v}\in\mathbb{R}^{K\times K}$ such that
$\hat{\bm{v}}^{\top}\hat{\bm{v}}+\hat{\bm{z}}^{\top}\hat{\bm{z}}=I_{K}$ and
the following hold for any constant $\epsilon\in(0,1/2)$:

$\displaystyle\hat{\bm{C}}=\tilde{\bm{C}}\hat{\bm{v}}+\bm{Q}\hat{\bm{z}},\quad\lVert\hat{\bm{v}}-\bm{v}\rVert_{2},\lVert\hat{\bm{z}}-p^{-1}\sum_{g=1}^{p}\bm{Q}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}(n^{1/2}\bm{\ell}_{g})^{\top}\bm{\Lambda}^{-1}\bm{v}\rVert_{2}=O_{P}(\lambda^{-1+\epsilon}).$
(S8.19)

###### Proof.

The expression for $\hat{\bm{C}}$ is a direct consequence of Lemma S8.3. By
Lemma S8.3,
$\lVert\hat{\bm{v}}-\bm{v}\rVert_{2}=O(\lVert\hat{\bm{z}}\rVert_{2}^{2})$ for
$\bm{v}=\hat{\bm{A}}\hat{\bm{B}}^{\top}$ and
$\hat{\bm{A}},\hat{\bm{B}}\in\mathbb{R}^{K\times K}$ the left and right
singular vectors of $\hat{\bm{v}}$. For $t\in[0,1]$, let
$\hat{\bm{z}}(t)=t\hat{\bm{z}}$,
$\hat{\bm{v}}(t)=\bm{v}\\{I_{K}-\hat{\bm{z}}(t)^{\top}\hat{\bm{z}}(t)\\}^{1/2}$,
and
$\bm{\gamma}(t)=(\hat{\bm{v}}(t)^{\top},\,\hat{\bm{z}}(t)^{\top})^{\top}\in\mathbb{R}^{(n-d)\times
K}$. Since
$\hat{\bm{v}}^{\top}\hat{\bm{v}}+\hat{\bm{z}}^{\top}\hat{\bm{z}}=I_{K}$,
$\hat{\bm{z}}$,
$\hat{\bm{v}}=\bm{v}(I_{K}-\hat{\bm{z}}^{\top}\hat{\bm{z}})^{1/2}$, meaning
$\bm{\gamma}(0)=(\bm{v}^{\top},\,\bm{0})^{\top}$,
$\bm{\gamma}(1)=(\hat{\bm{v}}^{\top},\,\hat{\bm{z}}^{\top})^{\top}$, and for
$\hat{\bm{C}}(t)=\bm{C}\hat{\bm{v}}(t)+\bm{Q}\hat{\bm{z}}(t)$,

$\displaystyle\lVert\bm{\gamma}(t)-\bm{\gamma}(0)\rVert_{2}=\lVert\hat{\bm{C}}(t)-\bm{C}\bm{v}\rVert_{2}\leq
c_{1}\lVert\hat{\bm{C}}-\bm{C}\bm{v}\rVert_{2}\leq c_{2}\lVert
P_{\hat{\bm{C}}}-P_{\bm{C}}\rVert_{2},\quad t\in[0,1]$

for some constants $c_{1},c_{2}>0$. By Taylor’s Theorem

$\displaystyle\bm{0}=\operatorname{vec}[\tilde{\bm{s}}\\{\bm{\gamma}(1)\\}]=\operatorname{vec}[\tilde{\bm{s}}\\{\bm{\gamma}(0)\\}]+\smallint_{0}^{1}\tilde{\bm{H}}\\{\bm{\gamma}(t)\\}\nabla_{t}\operatorname{vec}\\{\bm{\gamma}(t)\\}\text{d}t$
$\displaystyle\sum_{t\in[0,1]}\left\lVert\nabla_{t}\operatorname{vec}\\{\bm{\gamma}(t)\\}\right\rVert_{2}\leq\left\lVert\hat{\bm{z}}\right\rVert_{F}[1+\left\lVert\hat{\bm{z}}\right\rVert_{F}^{2}\\{1+o_{P}(1)\\}],$
(S8.20)

where $\left\lVert\hat{\bm{z}}\right\rVert_{F}=O_{P}(n^{-\eta})$ for any
$\eta\in(0,1/4)$ by Corollary S8.2 and Lemma S8.3. Then by the expression for
$\tilde{\bm{s}}$ in (S8.12),

$\displaystyle\operatorname{vec}\left\\{\begin{pmatrix}\bm{0}_{K\times K}\\\
-\tilde{\bm{s}}_{z}\end{pmatrix}\right\\}=\tilde{\bm{H}}^{*}\operatorname{vec}(\hat{\bm{z}})+\smallint_{0}^{1}[\tilde{\bm{H}}\\{\bm{\gamma}(t)\\}-\tilde{\bm{H}}^{*}]\nabla_{t}\operatorname{vec}\\{\bm{\gamma}(t)\\}\text{d}t+\operatorname{vec}\left\\{\begin{pmatrix}\bm{0}_{K\times
K}\\\ \hat{\bm{\xi}}\end{pmatrix}\right\\}$
$\displaystyle\tilde{\bm{s}}_{z}=(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{Q}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}(n^{1/2}\bm{\ell}_{g})^{\top}\bm{v},\quad\tilde{\bm{H}}^{*}=-(\lambda^{-1}\bm{v}^{\top}\bm{\Lambda}\bm{v})\otimes(\bm{0}_{K\times
K}\oplus I_{n-d-K})$

for any $\epsilon>0$, where
$\lVert\hat{\bm{\xi}}\rVert_{2}=O_{P}(\lambda^{-1+\epsilon})$ by Lemmas S8.17
and S8.19. Corollary S8.3 and (S8.20) imply

$\displaystyle\left\lVert\smallint_{0}^{1}[\tilde{\bm{H}}\\{\bm{\gamma}(t)\\}-\tilde{\bm{H}}^{*}]\nabla_{t}\operatorname{vec}\\{\bm{\gamma}(t)\\}\text{d}t\right\rVert_{2}=o_{P}(\lVert\hat{\bm{z}}\rVert_{2}),$

where an application of Lemma S8.18 then implies
$\lVert\hat{\bm{z}}\rVert_{2}=O_{P}(\lambda^{-1/2+\epsilon})$ for any
$\epsilon>0$. An application Lemma S8.3 and further applications of Corollary
S8.3 and (S8.20) complete the proof. ∎

######  Corollary S8.4.

Suppose the assumptions of Theorem S8.1 hold. Then the conclusions of Theorem
5.1 hold.

###### Proof.

This is a direct consequence of Theorem S8.1 and Lemma S8.3. ∎

######  Corollary S8.5.

Suppose the assumptions of Theorem S8.1 hold. Then
$\lVert\hat{\bm{C}}-\tilde{\bm{C}}\bm{v}\rVert_{\infty}=O_{P}(\lambda^{-1+\epsilon})$
for any $\epsilon>0$.

###### Proof.

Let $\bm{Z}=n^{-1/2}[\bm{C},\bm{X}]$,
$\bar{\bm{\ell}}_{g}=\bm{v}^{\top}(\lambda^{-1}\bm{\Lambda})^{-1}\bm{\ell}_{g}$,
and $\bm{\Delta}=\hat{\bm{z}}-(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{Q}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\bar{\bm{\ell}}_{g}^{\top}$.
Then (S8.19) in Theorem S8.1 implies

$\displaystyle\begin{aligned}
\lVert\hat{\bm{C}}-\tilde{\bm{C}}\bm{v}\rVert_{\infty}\leq&\lVert\tilde{\bm{C}}(\hat{\bm{v}}-\bm{v})\rVert_{\infty}+\lVert\bm{Q}\bm{\Delta}\rVert_{\infty}+\sum_{k=1}^{K}\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{W}_{g}\bm{e}_{g}\bar{\bm{\ell}}_{gk}\rVert_{\infty}\\\
&+\lVert\bm{Z}(\bm{Z}^{\top}\bm{Z})^{-1}(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{Z}^{\top}\bm{W}_{g}\bm{e}_{g}\bar{\bm{\ell}}_{g}^{\top}\rVert_{\infty}\\\
&+\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}(\bm{W}_{g}-I_{n})\bm{X}(\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{e}_{g}\bar{\bm{\ell}}_{g}^{\top}\rVert_{\infty}\\\
&+\lVert\bm{Z}(\bm{Z}^{\top}\bm{Z})^{-1}(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{Z}^{\top}(\bm{W}_{g}-I_{n})\bm{X}(\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{e}_{g}\bar{\bm{\ell}}_{g}^{\top}\rVert_{\infty},\end{aligned}$
(S8.21)

where since $\tilde{\bm{C}}(\hat{\bm{v}}-\bm{v})$ and $\bm{Q}\bm{\Delta}$ are
at most rank $2K$,

$\displaystyle\lVert\tilde{\bm{C}}(\hat{\bm{v}}-\bm{v})\rVert_{\infty}=O\\{\lVert\tilde{\bm{C}}(\hat{\bm{v}}-\bm{v})\rVert_{2}\\}=O_{P}(\lambda^{-1+\epsilon}),\quad\lVert\bm{Q}\bm{\Delta}\rVert_{\infty}=O(\lVert\bm{Q}\bm{\Delta}\rVert_{2})=O_{P}(\lambda^{-1+\epsilon})$

for any $\epsilon>0$ by Theorem S8.1. Similarly, since the fourth and sixth
matrices to the right of the inequality in (S8.21) are at most rank $K$,

$\displaystyle\lVert\bm{Z}(\bm{Z}^{\top}\bm{Z})^{-1}(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{Z}^{\top}\bm{W}_{g}\bm{e}_{g}\bar{\bm{\ell}}_{g}^{\top}\rVert_{\infty}=O_{P}\\{\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{Z}^{\top}\bm{W}_{g}\bm{e}_{g}\bar{\bm{\ell}}_{g}^{\top}\rVert_{2}\\}$
$\displaystyle\lVert\bm{Z}(\bm{Z}^{\top}\bm{Z})^{-1}(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{Z}^{\top}(\bm{W}_{g}-I_{n})\bm{X}(\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{e}_{g}\bar{\bm{\ell}}_{g}^{\top}\rVert_{\infty}$
$\displaystyle=$ $\displaystyle O_{P}\\{\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{Z}^{\top}(\bm{W}_{g}-I_{n})\bm{X}(\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{e}_{g}\bar{\bm{\ell}}_{g}^{\top}\rVert_{2}\\}.$

To derive the asymptotic properties of these Euclidean norms, we first see
that for some constants $c_{1},c_{2}>0$ and
$\tilde{\bm{c}}_{i}=(\bm{C}_{i\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}}^{\top},\bm{X}_{i\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}}^{\top})^{\top}$,

$\displaystyle(\lambda
p)^{-1/2}\sum_{g=1}^{p}\bm{Z}^{\top}\bm{W}_{g}\bm{e}_{g}\bar{\bm{\ell}}_{g}^{\top}=(\lambda
p)^{-1/2}\sum_{g=1}^{p}\bm{Z}^{\top}\bm{e}_{g}\bar{\bm{\ell}}_{g}^{\top}+(\lambda
p)^{-1/2}\sum_{g=1}^{p}\bm{Z}^{\top}(\bm{W}_{g}-I_{n})\bm{e}_{g}\bar{\bm{\ell}}_{g}^{\top}$
$\displaystyle\operatorname{\mathbb{V}}\\{(\lambda
p)^{-1/2}\sum_{g=1}^{p}\bm{Z}^{\top}(\bm{W}_{g}-I_{n})\bm{e}_{g}\bar{\bm{\ell}}_{gk}\\}\leq
c_{1}p^{-1}\sum_{g=1}^{p}n^{-1}\sum_{i=1}^{n}\operatorname{\mathbb{E}}\\{(w_{gi}-1)^{2}\bm{e}_{gi}^{2}\tilde{\bm{c}}_{i}\tilde{\bm{c}}_{i}^{\top}\\}\preceq
c_{2}I_{K},\quad k\in[K],$

where $\lVert(\lambda
p)^{-1/2}\sum_{g=1}^{p}\bm{Z}^{\top}\bm{e}_{g}\bar{\bm{\ell}}_{g}^{\top}\rVert_{2}=O_{P}(1)$
by Lemma S8.2. This implies the fourth term in (S8.21) is
$O_{P}(\lambda^{-1})$. Next, Lemma S8.4 and Corollary S8.1 imply that for some
constant $c>0$ and any $\epsilon>0$,

$\displaystyle\max_{g\in[p]}\lVert\bm{Z}^{\top}(\bm{W}_{g}-I_{n})(n^{-1/2}\bm{X})\rVert_{2}=$
$\displaystyle
O_{P}(n^{-1/2+\epsilon}),\quad\max_{g\in[p]}\lVert\\{\rVert(\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}\\}_{2}\leq
c\\{1+o_{P}(1)\\}$ $\displaystyle\max_{g\in[p]}\lVert(\lambda
n)^{-1/2}\bm{X}^{\top}\bm{W}_{g}\bm{e}_{g}\bar{\bm{\ell}}_{g}\rVert_{2}\leq$
$\displaystyle c\\{\max_{g\in[p]}\lVert
n^{-1/2}\bm{X}^{\top}\bm{e}_{g}\rVert_{2}+\max_{g\in[p]}\lVert
n^{-1/2}\bm{X}^{\top}(\bm{W}_{g}-I_{n})\bm{e}_{g}\rVert_{2}\\}$
$\displaystyle=$ $\displaystyle O_{P}(n^{\epsilon}),$

which implies the sixth term in (S8.21) is $O_{P}(\lambda^{-1+\epsilon})$ for
any $\epsilon>0$. We next consider the third term in (S8.21). For $i\in[n]$
and $k\in[K]$, the $i$th element of the third vector can be expressed as

$\displaystyle
x_{ik}=p^{-1}\lambda^{-1/2}\sum_{g=1}^{p}w_{gi}\bm{e}_{gi}(\lambda^{-1/2}\bar{\bm{\ell}}_{gk})=p^{-1/2}\lambda^{-1/2}(p^{-1/2}\sum_{g=1}^{p}a_{gi}b_{gk})$
$\displaystyle a_{gi}=w_{gi}\bm{e}_{gi},\quad\lvert b_{gk}\rvert\leq c$

for some constant $c>0$. Since $a_{1i},\ldots,a_{pi}$ are independent and mean
0, Lemma S8.4 and Corollary S8.1 imply $\max_{i\in[n]}\lvert
x_{ik}\rvert=O_{P}(\lambda^{-1+\epsilon})$ for any $\epsilon>0$, which implies
the third term in (S8.21) is $O_{P}(\lambda^{-1+\epsilon})$. For the fifth and
final term in (S8.21), assume without loss of generality that
$n^{-1}\bm{X}^{\top}\bm{X}=I_{d}$. Then the fifth term in (S8.21) can be
bounded above by

$\displaystyle\begin{aligned} &\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}(\bm{W}_{g}-I_{n})(n^{-1/2}\bm{X})\\{(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}-I_{d}\\}(n^{-1/2}\bm{X})^{\top}\bm{W}_{g}\bm{e}_{g}\bar{\bm{\ell}}_{g}^{\top}\rVert_{\infty}\\\
&+\sum_{k=1}^{K}\sum_{j=1}^{d}\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}(\bm{W}_{g}-I_{n})(n^{-1/2}\bm{X}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}j})(n^{-1/2}\bm{X}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}j})^{\top}\bm{W}_{g}\bm{e}_{g}\bar{\bm{\ell}}_{gk}^{\top}\rVert_{\infty}.\end{aligned}$
(S8.22)

First, since the first matrix is at most rank $K$, and
$\max_{g\in[p]}\lVert(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}-I_{K}\rVert_{2}=O_{P}(n^{-1/2+\epsilon})$
for any $\epsilon>0$,

$\displaystyle\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}(\bm{W}_{g}-I_{n})(n^{-1/2}\bm{X})\\{(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}-I_{K}\\}(n^{-1/2}\bm{X})^{\top}\bm{W}_{g}\bm{e}_{g}\bar{\bm{\ell}}_{g}^{\top}\rVert_{\infty}$
$\displaystyle\leq$ $\displaystyle c\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}(\bm{W}_{g}-I_{n})(n^{-1/2}\bm{X})\\{(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}-I_{K}\\}(n^{-1/2}\bm{X})^{\top}\bm{W}_{g}\bm{e}_{g}\bar{\bm{\ell}}_{g}^{\top}\rVert_{2}=O_{P}(\lambda^{-1+\epsilon})$

for some constant $c>0$. Next, for fixed $j\in[d]$ and $k\in[K]$, the $i$th
element of the second matrix in (S8.22) can be expressed as

$\displaystyle(\lambda
n)^{-1/2}p^{-1}\sum_{g=1}^{p}(w_{gi}-1)\bm{X}_{ij}(n^{-1/2}\bm{X}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}j}^{\top}\bm{W}_{g}\bm{e}_{g})(\lambda^{-1/2}\bar{\bm{\ell}}_{gk})=(\lambda
n)^{-1/2}a_{ijk},\quad i\in[n],$

where for some constant $c>0$,

$\displaystyle\max_{i\in[n]}\lvert a_{ijk}\rvert\leq
c(\max_{i\in[n],g\in[p]}\lvert w_{gi}-1\rvert)(\max_{g\in[p]}\lvert
n^{-1/2}\bm{X}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}j}^{\top}\bm{W}_{g}\bm{e}_{g}\rvert)=O_{P}(n^{\epsilon})$

for some any constant $\epsilon>0$, which completes the proof. ∎

######  Corollary S8.6.

Suppose the assumptions of Theorem S8.1 hold and let $\hat{\bm{C}}$ and
$\bm{v}$ be as defined in the statement of Theorem S8.1. Then for any
$\epsilon>0$,

$\displaystyle\max_{g\in[p]}\lVert(\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\hat{\bm{C}})(\bm{v}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\bm{v})^{-1}-I_{K}\rVert_{2}=O_{P}(\lambda^{-1+\epsilon})$
(S8.23a)
$\displaystyle\max_{g\in[p]}\lVert\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\hat{\bm{C}}-I_{K}\rVert_{2}=O_{P}(n^{-1/2+\epsilon}).$
(S8.23b)

###### Proof.

Let
$\bm{A}_{g}=\bm{v}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\bm{v}$
and $\hat{\bm{v}}$, $\hat{\bm{z}}$ be as defined in Theorem S8.1. We can
express $\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\hat{\bm{C}}$ as

$\displaystyle\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\hat{\bm{C}}=$
$\displaystyle\hat{\bm{v}}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\hat{\bm{v}}+\hat{\bm{z}}^{\top}\bm{Q}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\hat{\bm{v}}+(\hat{\bm{z}}^{\top}\bm{Q}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\hat{\bm{v}})^{\top}+\hat{\bm{z}}^{\top}\bm{Q}^{\top}\bm{P}_{g}^{\perp}\bm{Q}\hat{\bm{z}}.$

By (S8.7) in Lemma S8.9,
$\max_{g\in[p]}\lVert\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\rVert_{2}=1+O_{P}(n^{-1/2+\epsilon})$
for any $\epsilon>0$. Therefore,

$\displaystyle\max_{g\in[p]}\left\lVert\hat{\bm{z}}^{\top}\bm{Q}^{\top}\bm{P}_{g}^{\perp}\bm{Q}\hat{\bm{z}}\bm{A}_{g}^{-1}\right\rVert_{2}\leq\left\lVert\hat{\bm{z}}\right\rVert_{2}^{2}\max_{g\in[p]}\lVert\bm{A}_{g}^{-1}\rVert_{2}\max_{g\in[p]}\left\lVert\bm{W}_{g}\right\rVert_{2}=O_{P}(\lambda^{-1+\epsilon})$

for any $\epsilon>0$ by Theorem S8.1. Next,

$\displaystyle\lVert\hat{\bm{v}}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\hat{\bm{v}}\bm{A}_{g}^{-1}-I_{K}\rVert_{2}\leq
2\left\lVert\hat{\bm{v}}-\bm{v}\right\rVert_{2}\lVert\bm{A}_{g}\rVert_{2}\lVert\bm{A}_{g}^{-1}\rVert_{2}+\left\lVert\hat{\bm{v}}-\bm{v}\right\rVert_{2}^{2}\lVert\bm{A}_{g}^{-1}\rVert_{2}\lVert\bm{A}_{g}\rVert_{2}.$

Since
$\left\lVert\hat{\bm{v}}-\bm{v}\right\rVert_{2}=O_{P}(\lambda^{-1+\epsilon})$
for any $\epsilon>0$ by Theorem S8.1,
$\max_{g\in[p]}\lVert\hat{\bm{v}}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\hat{\bm{v}}\bm{A}_{g}^{-1}-I_{K}\rVert_{2}=O_{P}(\lambda^{-1+\epsilon})$.
Next, let $\bm{s}=(\lambda
p)^{-1}\sum_{g=1}^{p}\bm{P}_{g}^{\perp}\bm{e}_{g}(n^{1/2}\bm{\ell}_{g})^{\top}(\lambda^{-1}\bm{\Lambda})^{-1}$.
Then

$\displaystyle\lVert\hat{\bm{z}}^{\top}\bm{Q}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\hat{\bm{v}}\rVert_{2}\leq\lVert\bm{s}^{\top}\bm{Q}\bm{Q}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\rVert_{2}+\lVert\bm{v}^{\top}\bm{s}^{\top}\bm{Q}-\hat{\bm{z}}\rVert_{2}\lVert\bm{P}_{g}^{\perp}\tilde{\bm{C}}\rVert_{2},$

where $\max_{g\in[p]}\lVert\bm{P}_{g}^{\perp}\tilde{\bm{C}}\rVert_{2}\leq
c\\{1+o_{P}(1)\\}$ for some constant $c>0$ by the proof of Lemma S8.9.
Therefore,

$\displaystyle\max_{g\in[p]}(\lVert\bm{v}^{\top}\bm{s}^{\top}\bm{Q}-\hat{\bm{z}}\rVert_{2}\lVert\bm{P}_{g}^{\perp}\tilde{\bm{C}}\rVert_{2})\leq
c\\{1+o_{P}(1)\\}\lVert\bm{v}^{\top}\bm{s}^{\top}\bm{Q}-\hat{\bm{z}}\rVert_{2}=O_{P}(\lambda^{-1+\epsilon})$

by Theorem S8.1. Since an application of Lemma S8.9 and (S8.23a) imply
(S8.23b), we need only show that
$\max_{g\in[p]}\lVert\bm{s}^{\top}\bm{Q}\bm{Q}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\rVert_{2}=O_{P}(\lambda^{-1+\epsilon})$
for any $\epsilon>0$ to complete the proof.

Define $\bm{H}=\bm{Q}\bm{Q}^{\top}=P_{[\bm{X},\tilde{\bm{C}}]}^{\perp}$. Then
for any $r,t\in[K]$,

$\displaystyle\begin{aligned}
&\bm{s}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}r}^{\top}\bm{Q}\bm{Q}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}=(\lambda
p)^{-1}\bar{\bm{\ell}}_{gr}\bm{e}_{g}^{\top}\bm{P}_{g}^{\perp}\bm{H}\bm{P}_{g}^{\perp}\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}+(\lambda
p)^{-1}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}\bm{e}_{h}^{\top}\bm{P}_{h}^{\perp}\bm{H}\bm{P}_{g}^{\perp}\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}\\\
&\bar{\bm{\ell}}_{hr}=n^{1/2}\lambda^{-1}\bm{\Lambda}_{r\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}}^{\top}\bm{\ell}_{h},\quad
h\in[p],\end{aligned}$ (S8.24)

where $\lvert\bar{\bm{\ell}}_{hr}\rvert\leq c\lambda^{1/2}$ for some constant
$c>0$. Therefore,

$\displaystyle\max_{g\in[p]}\lvert(\lambda
p)^{-1}\bar{\bm{\ell}}_{gr}\bm{e}_{g}^{\top}\bm{P}_{g}^{\perp}\bm{H}\bm{P}_{g}^{\perp}\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}\rvert\leq
c(\lambda n)^{-1/2}(\max_{g\in[p]}\lVert
n^{-1/2}\bm{e}_{g}\rVert_{2})(\max_{g\in[p]}\lVert\bm{W}_{g}^{2}\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}\rVert_{2})$

for some constant $c>0$. It is easy to see that $\max_{g\in[p]}\lVert
n^{-1/2}\bm{e}_{g}\rVert_{2}=O_{P}(n^{\epsilon})$ and
$\max_{g\in[p]}\allowbreak\lVert\bm{W}_{g}^{2}\allowbreak\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}\rVert_{2}=O_{P}(1)$,
where the latter follows from Corollary S8.1 and implies

$\displaystyle\max_{g\in[p]}\lvert(\lambda
p)^{-1}\bar{\bm{\ell}}_{gr}\bm{e}_{g}^{\top}\bm{P}_{g}^{\perp}\bm{H}\bm{P}_{g}^{\perp}\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}\rvert=O_{P}(\lambda^{-1+\epsilon})$

for any $\epsilon>0$. For the second term in (S8.24),

$\displaystyle\begin{aligned} &(\lambda p)^{-1}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}\bm{e}_{h}^{\top}\bm{P}_{h}^{\perp}\bm{H}\bm{P}_{g}^{\perp}\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}=(\lambda
p)^{-1}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}\bm{e}_{h}^{\top}\bm{W}_{h}\bm{H}(\bm{W}_{g}-I_{n})\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}\\\
&-(\lambda p)^{-1}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}\bm{e}_{h}^{\top}\bm{W}_{h}\bm{X}(\bm{X}^{\top}\bm{W}_{h}\bm{X})^{-1}\bm{X}^{\top}(\bm{W}_{h}-I_{n})\bm{H}(\bm{W}_{g}-I_{n})\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}\\\
&-(\lambda p)^{-1}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}\bm{e}_{h}^{\top}\bm{W}_{h}\bm{H}(\bm{W}_{g}-I_{n})\bm{X}(\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}\bm{X}^{\top}(\bm{W}_{g}-I_{n})\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}\\\
&+(\lambda p)^{-1}\sum_{h\neq
g}\\{\bar{\bm{\ell}}_{hr}\bm{e}_{h}^{\top}\bm{W}_{h}\bm{X}(\bm{X}^{\top}\bm{W}_{h}\bm{X})^{-1}\bm{X}^{\top}(\bm{W}_{h}-I_{n})\bm{H}(\bm{W}_{g}-I_{n})\bm{X}(\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}\\\
&\times\bm{X}^{\top}(\bm{W}_{g}-I_{n})\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}\\}.\end{aligned}$
(S8.25)

Define $\bm{R}=(n^{-1}\bm{C}^{\top}P_{\bm{X}}^{\perp}\bm{C})^{-1/2}$ and
$\bm{Z}=[\bm{C},\bm{X}]$, where we assume without loss of generality that
$n^{-1}\bm{X}^{\top}\bm{X}=I_{d}$. Then the first term in (S8.25) can be
expressed as

$\displaystyle\begin{aligned} &(\lambda p)^{-1}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}\bm{e}_{h}^{\top}\bm{W}_{h}\bm{H}(\bm{W}_{g}-I_{n})\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}=(\lambda
p)^{-1}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}\bm{e}_{h}^{\top}(\bm{W}_{g}-I_{n})(n^{-1/2}\bm{C})\bm{R}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}\\\
&+(\lambda p)^{-1}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}\bm{e}_{h}^{\top}(\bm{W}_{h}-I_{n})(\bm{W}_{g}-I_{n})(n^{-1/2}\bm{C})\bm{R}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}\\\
&-(\lambda p)^{-1}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}\bm{e}_{h}^{\top}(\bm{W}_{g}-I_{n})(n^{-1/2}\bm{X})(n^{-1}\bm{X}^{\top}\bm{C}\bm{R}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t})\\\
&-(\lambda p)^{-1}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}\bm{e}_{h}^{\top}(\bm{W}_{h}-I_{n})(\bm{W}_{g}-I_{n})(n^{-1/2}\bm{X})(n^{-1}\bm{X}^{\top}\bm{C}\bm{R}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t})\\\
&-(\lambda p)^{-1}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}\bm{e}_{h}^{\top}\bm{W}_{h}(n^{-1/2}\bm{Z})(n^{-1}\bm{Z}^{\top}\bm{Z})\\{n^{-1}\bm{Z}^{\top}(\bm{W}_{g}-I_{n})P_{\bm{X}}^{\perp}\bm{C}\\}\bm{R}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}.\end{aligned}$
(S8.26)

Define $\bm{x}_{g}=(\lambda p)^{-1/2}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}\bm{e}_{h}$. Since
$\lvert\lambda^{-1/2}\bar{\bm{\ell}}_{hr}\rvert\leq c$ for some constant
$c>0$, $\operatorname{\mathbb{E}}(\bm{x}_{g_{i}}^{2m})\leq c_{m}$ for all
$i\in[n]$ for some constant $c_{m}>0$ that only depends on the positive
integer $m$ by Lemma S8.4. Since the rows of
$(\bm{W}_{g}-I_{n})(n^{-1/2}\bm{C})$ are mean $\bm{0}$ and independent
conditional on $\\{\bm{C},\bm{x}_{g}\\}$, Corollary S8.1 implies

$\displaystyle\max_{g\in[p]}\lvert(\lambda p)^{-1/2}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}\bm{e}_{h}^{\top}(\bm{W}_{g}-I_{n})(n^{-1/2}\bm{C}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}k})\rvert=O_{P}(n^{\epsilon}),\quad
k\in[K]$

for any $\epsilon>0$, which then implies

$\displaystyle\max_{g\in[p]}\lvert(\lambda p)^{-1}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}\bm{e}_{h}^{\top}(\bm{W}_{g}-I_{n})(n^{-1/2}\bm{C})\bm{R}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}\rvert=O_{P}(\lambda^{-1+\epsilon})$

for any $\epsilon>0$. Identical analyses and repeated applications of Lemma
S8.4 and Corollary S8.1 can be used to show that the maximum, over $g\in[p]$,
absolute value of the remaining four terms in (S8.26) are all
$O_{P}(\lambda^{-1+\epsilon})$, which shows that the maximum, over $g\in[p]$,
absolute value of the first term in (S8.25) is $O_{P}(\lambda^{-1+\epsilon})$
for any $\epsilon>0$. For the second term in (S8.25), we see that for $h\neq
g$,

$\displaystyle\begin{aligned}
&n^{-1/2}\bm{X}^{\top}(\bm{W}_{h}-I_{n})\bm{H}(\bm{W}_{g}-I_{n})\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}=n^{-1}\bm{X}^{\top}(\bm{W}_{h}-I_{n})(\bm{W}_{g}-I_{n})\bm{C}\bm{R}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}\\\
&-\\{n^{-1}\bm{X}^{\top}(\bm{W}_{h}-I_{n})(\bm{W}_{g}-I_{n})\bm{X}\\}(n^{-1}\bm{X}^{\top}\bm{C}\bm{R}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t})\\\
&-\\{n^{-1}\bm{X}^{\top}(\bm{W}_{h}-I_{n})\bm{Z}\\}(n^{-1}\bm{Z}^{\top}\bm{Z})\\{n^{-1}\bm{Z}^{\top}(\bm{W}_{g}-I_{n})\bm{C}\bm{R}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}\\}\\\
&+\\{n^{-1}\bm{X}^{\top}(\bm{W}_{h}-I_{n})\bm{Z}\\}(n^{-1}\bm{Z}^{\top}\bm{Z})\\{n^{-1}\bm{Z}^{\top}(\bm{W}_{g}-I_{n})\bm{X}\\}(n^{-1}\bm{X}^{\top}\bm{C}\bm{R}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}).\end{aligned}$
(S8.27)

Since the diagonal entries of $(\bm{W}_{h}-I_{n})(\bm{W}_{g}-I_{n})$ are
independent and mean 0, Corollary S8.1 implies the terms in (S8.27) satisfy

$\displaystyle\max_{g\neq h\in[p]\times[p]}\lVert
n^{-1}\bm{X}^{\top}(\bm{W}_{h}-I_{n})(\bm{W}_{g}-I_{n})\bm{C}\bm{R}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}\rVert_{2}=O_{P}(n^{-1/2+\epsilon})$
$\displaystyle\max_{g\neq
h\in[p]\times[p]}\lVert\\{n^{-1}\bm{X}^{\top}(\bm{W}_{h}-I_{n})(\bm{W}_{g}-I_{n})\bm{X}\\}(n^{-1}\bm{X}^{\top}\bm{C}\bm{R}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t})\rVert_{2}=O_{P}(n^{-1/2+\epsilon})$
$\displaystyle\max_{g\neq
h\in[p]\times[p]}\lVert\\{n^{-1}\bm{X}^{\top}(\bm{W}_{h}-I_{n})\bm{Z}\\}(n^{-1}\bm{Z}^{\top}\bm{Z})\\{n^{-1}\bm{Z}^{\top}(\bm{W}_{g}-I_{n})\bm{C}\bm{R}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}\\}\rVert_{2}=O_{P}(n^{-1+\epsilon})$
$\displaystyle\max_{g\neq
h\in[p]\times[p]}\lVert\\{n^{-1}\bm{X}^{\top}(\bm{W}_{h}-I_{n})\bm{Z}\\}(n^{-1}\bm{Z}^{\top}\bm{Z})\\{n^{-1}\bm{Z}^{\top}(\bm{W}_{g}-I_{n})\bm{X}\\}(n^{-1}\bm{X}^{\top}\bm{C}\bm{R}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t})\rVert_{2}=O_{P}(n^{-1+\epsilon}),$

which implies $\max_{g\neq h\in[p]\times[p]}\lVert
n^{-1/2}\bm{X}^{\top}(\bm{W}_{h}-I_{n})\bm{H}(\bm{W}_{g}-I_{n})\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}\rVert_{2}=O_{P}(n^{-1/2+\epsilon})$
for any $\epsilon>0$. Next, for some constant $c>0$,

$\displaystyle\lVert\lambda^{-1/2}\bar{\bm{\ell}}_{hr}n^{-1/2}\bm{e}_{h}^{\top}\bm{W}_{h}\bm{X}\rVert_{2}\leq
c\\{\lVert n^{-1/2}\bm{e}_{h}^{\top}\bm{X}\rVert_{2}+\lVert
n^{-1/2}\bm{e}_{h}^{\top}(\bm{W}_{h}-I_{n})\bm{X}\rVert_{2}\\},$

where, for any $\epsilon>0$,

$\displaystyle\max_{h\in[p]}\lVert
n^{-1/2}\bm{e}_{h}^{\top}(\bm{W}_{h}-I_{n})\bm{X}\rVert_{2},\,\max_{h\in[p]}\lVert
n^{-1/2}\bm{e}_{h}^{\top}\bm{X}\rVert_{2}=O_{P}(n^{\epsilon})$

by Corollary S8.1 and because $\bm{e}_{h}$ is sub-Gaussian, respectively.
Therefore, the second term in (S8.25) satisfies

$\displaystyle\max_{g\in[p]}\lvert(\lambda p)^{-1}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}\bm{e}_{h}^{\top}\bm{W}_{h}\bm{X}(\bm{X}^{\top}\bm{W}_{h}\bm{X})^{-1}\bm{X}^{\top}(\bm{W}_{h}-I_{n})\bm{H}(\bm{W}_{g}-I_{n})\tilde{\bm{C}}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}t}\rvert$
$\displaystyle=O_{P}\\{(\lambda n)^{-1/2+\epsilon}\\}$
$\displaystyle=O_{P}(\lambda^{-1+\epsilon})$

for any $\epsilon>0$. Identical techniques to those used to derive the
properties of the second term in (S8.25) can also be used to show that the
maximum, over $g\in[p]$, absolute values of the third and fourth terms in
(S8.25) are $O_{P}(\lambda^{-1+\epsilon})$ for any $\epsilon>0$. The details
have been omitted. ∎

######  Corollary S8.7.

Suppose the assumptions of Theorem S8.1 hold, let
$\hat{\bm{Z}}=[\hat{\bm{C}},\bm{X}]$ for $\hat{\bm{C}}$ as defined in the
statement of Theorem S8.1, and define $\hat{\bm{\ell}}_{g}$ to be the first
$K$ elements of the $K+d$ vector
$(\hat{\bm{Z}}^{\top}\bm{W}_{g}\hat{\bm{Z}})^{-1}\hat{\bm{Z}}^{\top}\bm{W}_{g}\bm{y}_{g}$.
Then $\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\hat{\bm{\ell}}_{g}\hat{\bm{\ell}}_{g}^{\top}-(\lambda
p)^{-1}\bm{v}^{\top}\sum_{g=1}^{p}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\bm{v}\rVert_{2}=O_{P}(\lambda^{-1+\epsilon})$
and $\lVert(\lambda
p)^{-1}\sum_{g=1}^{p}\hat{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}-(\lambda
p)^{-1}\bm{v}^{\top}\sum_{g=1}^{p}\tilde{\bm{\ell}}_{g}\tilde{\bm{\ell}}_{g}^{\top}\rVert_{2}=O_{P}(\lambda^{-1+\epsilon})$
for any $\epsilon>0$ and $\bm{v}$ as defined in the statement of Theorem S8.1.

###### Proof.

Let $\hat{\bm{v}}$ and $\hat{\bm{z}}$ be as defined in Theorem S8.1, and let
$\bm{s}$ and $\bm{H}$ be as defined in the proof of Corollary S8.6. We can
express $\hat{\bm{\ell}}_{g}$ as

$\displaystyle\begin{aligned}
\hat{\bm{\ell}}_{g}=&(\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\hat{\bm{C}})^{-1}\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{y}_{g}=(\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\hat{\bm{C}})^{-1}\hat{\bm{v}}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\tilde{\bm{\ell}}_{g}+(\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\hat{\bm{C}})^{-1}\hat{\bm{v}}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\\\
&+(\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\hat{\bm{C}})^{-1}\hat{\bm{z}}^{\top}\bm{Q}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\tilde{\bm{\ell}}_{g}+(\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\hat{\bm{C}})^{-1}\hat{\bm{z}}^{\top}\bm{Q}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\\\
=&\bm{v}^{\top}\tilde{\bm{\ell}}_{g}+\\{(\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\hat{\bm{C}})^{-1}(\hat{\bm{v}}^{\top}\bm{v})(\bm{v}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\bm{v})-I_{K}\\}\bm{v}^{\top}\tilde{\bm{\ell}}_{g}\\\
&+\bm{v}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}+\bm{v}^{\top}\\{(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}})^{-1}-I_{K}\\}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\\\
&+\\{(\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\hat{\bm{C}})^{-1}\hat{\bm{v}}^{\top}-(\bm{v}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\bm{v})^{-1}\bm{v}^{\top}\\}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\\\
&+(\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\hat{\bm{C}})^{-1}\bm{v}^{\top}\bm{s}^{\top}\bm{H}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\tilde{\bm{\ell}}_{g}+(\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\hat{\bm{C}})^{-1}\bm{\Delta}^{\top}\bm{Q}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\tilde{\bm{\ell}}_{g}\\\
&+(\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\hat{\bm{C}})^{-1}\bm{v}^{\top}\bm{s}^{\top}\bm{H}\bm{P}_{g}^{\perp}\bm{e}_{g}+\bm{\Delta}^{\top}\bm{Q}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\\\
&+\\{(\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\hat{\bm{C}})^{-1}-I_{K}\\}\bm{\Delta}^{\top}\bm{Q}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\end{aligned}$
(S8.28)

for $\bm{\Delta}=\hat{\bm{z}}-\bm{Q}^{\top}\bm{s}\bm{v}$. Lemmas S8.2 and
S8.9, Theorem S8.1, and Corollary S8.6 imply

$\displaystyle\begin{aligned}
&\max_{g\in[p]}\lVert\\{(\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\hat{\bm{C}})^{-1}(\hat{\bm{v}}^{\top}\bm{v})(\bm{v}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\bm{v})-I_{K}\\}\bm{v}^{\top}\tilde{\bm{\ell}}_{g}\rVert_{2}=O_{P}(\lambda^{-1/2+\epsilon})\\\
&\max_{g\in[p]}\lVert\bm{v}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\rVert_{2}=O_{P}(n^{\epsilon})\\\
&\max_{g\in[p]}\lVert\bm{v}^{\top}\\{(\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}})^{-1}-I_{K}\\}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\rVert_{2}=O_{P}(n^{-1/2+\epsilon})\\\
&\max_{g\in[p]}\lVert\\{(\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\hat{\bm{C}})^{-1}\hat{\bm{v}}^{\top}-(\bm{v}^{\top}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\bm{v})^{-1}\bm{v}^{\top}\\}\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\rVert_{2}=O_{P}(\lambda^{-1+\epsilon})\\\
&\max_{g\in[p]}\lVert(\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\hat{\bm{C}})^{-1}\bm{\Delta}^{\top}\bm{Q}^{\top}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\tilde{\bm{\ell}}_{g}\rVert_{2}=O_{P}(\lambda^{-1/2+\epsilon})\\\
&\max_{g\in[p]}\lVert\bm{\Delta}^{\top}\bm{Q}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\rVert_{2}=O_{P}(n^{-\delta})\\\
&\max_{g\in[p]}\lVert\\{(\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\hat{\bm{C}})^{-1}-I_{K}\\}\bm{\Delta}^{\top}\bm{Q}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}\rVert_{2}=O_{P}(\lambda^{-1+\epsilon})\end{aligned}$
(S8.29)

for $\delta>0$ sufficiently small and any $\epsilon>0$. The second line
follows from the fact that for
$\bm{R}=(n^{-1}\bm{C}^{\top}P_{\bm{X}}^{\perp}\bm{C})^{-1/2}$,

$\displaystyle\tilde{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\bm{e}_{g}=$
$\displaystyle\bm{R}(n^{-1/2}\bm{C}^{\top}P_{\bm{X}}^{\perp}\bm{e}_{g})+\bm{R}\\{n^{-1/2}\bm{C}^{\top}(\bm{W}_{g}-I_{n})\bm{e}_{g}\\}$
$\displaystyle-\bm{R}(n^{-1}\bm{C}^{\top}\bm{X})(n^{-1}\bm{X}^{\top}\bm{X})\\{n^{-1/2}\bm{X}^{\top}(\bm{W}_{g}-I_{n})\bm{e}_{g}\\}$
$\displaystyle-\\{n^{-1/2}\tilde{\bm{C}}^{\top}(\bm{W}_{g}-I_{n})\bm{X}\\}(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}(n^{-1/2}\bm{X}^{\top}\bm{W}_{g}\bm{e}_{g}),$

where

$\displaystyle\max_{g\in[p]}\lVert\bm{R}(n^{-1/2}\bm{C}^{\top}P_{\bm{X}}^{\perp}\bm{e}_{g})\rVert_{2}=O_{P}(\max_{g\in[p]}\lVert
n^{-1/2}\bm{C}^{\top}P_{\bm{X}}^{\perp}\bm{e}_{g}\rVert_{2})=O_{P}(n^{\epsilon})$
$\displaystyle\max_{g\in[p]}\lVert
n^{-1/2}\bm{C}^{\top}(\bm{W}_{g}-I_{n})\bm{e}_{g}\rVert_{2},\,\max_{g\in[p]}\lVert
n^{-1/2}\bm{X}^{\top}(\bm{W}_{g}-I_{n})\bm{e}_{g}\rVert_{2}=O_{P}(n^{\epsilon})$

by Lemma S8.2 and Corollary S8.1, respectively, and because

$\displaystyle\max_{g\in[p]}\lVert\\{n^{-1/2}\tilde{\bm{C}}^{\top}(\bm{W}_{g}-I_{n})\bm{X}\\}(n^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}(n^{-1/2}\bm{X}^{\top}\bm{W}_{g}\bm{e}_{g})\rVert_{2}=O_{P}(n^{-1/2+\epsilon})$

for any $\epsilon>0$. Identical techniques used to prove Corollary S8.6 can
also be used to show

$\displaystyle\max_{g\in[p]}\lVert(\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\hat{\bm{C}})^{-1}\bm{v}^{\top}\bm{s}^{\top}\bm{H}\bm{P}_{g}^{\perp}\tilde{\bm{C}}\tilde{\bm{\ell}}_{g}\rVert_{2}=O_{P}(\lambda^{-1/2+\epsilon})$
(S8.30)

for any $\epsilon>0$. Lastly,
$\max_{g\in[p]}\lVert(\hat{\bm{C}}^{\top}\bm{P}_{g}^{\perp}\hat{\bm{C}})^{-1}\bm{v}^{\top}\bm{s}^{\top}\bm{H}\bm{P}_{g}^{\perp}\bm{e}_{g}\rVert_{2}\leq\\{1+o_{P}(1)\\}\max_{g\in[p]}\lVert\bm{s}^{\top}\bm{H}\bm{P}_{g}^{\perp}\bm{e}_{g}\rVert_{2}$,
where for $\bar{\bm{\ell}}_{hr}$ as defined in (S8.24),

$\displaystyle\bm{s}_{\mathchoice{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\displaystyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\textstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptstyle*$}}}}}{\mathbin{\vbox{\hbox{\scalebox{1.0}{$\scriptscriptstyle*$}}}}}r}^{\top}\bm{H}\bm{P}_{g}^{\perp}\bm{e}_{g}=(\lambda
p)^{-1}\bar{\bm{\ell}}_{gr}\bm{e}_{g}^{\top}\bm{P}_{g}^{\perp}\bm{H}\bm{P}_{g}^{\perp}\bm{e}_{g}+(\lambda
p)^{-1}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}\bm{e}_{h}^{\top}\bm{P}_{h}^{\perp}\bm{H}\bm{P}_{g}^{\perp}\bm{e}_{g},\quad
r\in[K].$ (S8.31)

We first see that for some constant $c>0$, where

$\displaystyle\lvert(\lambda
p)^{-1}\bar{\bm{\ell}}_{gr}\bm{e}_{g}^{\top}\bm{P}_{g}^{\perp}\bm{H}\bm{P}_{g}^{\perp}\bm{e}_{g}\rvert\leq
c\lambda^{-1/2}\\{\max_{g\in[p]}(p^{-1}\bm{e}_{g}^{\top}\bm{e}_{g})\\}(\max_{g\in[p],i\in[n]}w_{gi}^{2})=O_{P}(\lambda^{-1/2+\epsilon})$

for any $\epsilon>0$. For $\bm{Z}=[\bm{C},\bm{X}]$, the second term in (S8.31)
can be expressed as

$\displaystyle\begin{aligned} &(\lambda p)^{-1}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}\bm{e}_{h}^{\top}\bm{P}_{h}^{\perp}\bm{H}\bm{P}_{g}^{\perp}\bm{e}_{g}=(\lambda
p)^{-1}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}\bm{e}_{h}^{\top}\bm{W}_{h}\bm{W}_{g}\bm{e}_{g}\\\
&-(\lambda p)^{-1}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}(n^{-1/2}\bm{e}_{h}^{\top}\bm{W}_{h}\bm{Z})(n^{-1}\bm{Z}^{\top}\bm{Z})(n^{-1/2}\bm{Z}^{\top}\bm{W}_{g}\bm{e}_{g})\\\
&-(\lambda p)^{-1}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}\bm{e}_{h}^{\top}\bm{W}_{h}\bm{H}(\bm{W}_{g}-I_{n})\bm{X}(\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}\bm{X}^{\top}\bm{W}_{g}\bm{e}_{g}\\\
&-(\lambda p)^{-1}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}\bm{e}_{h}^{\top}\bm{W}_{h}\bm{X}(\bm{X}^{\top}\bm{W}_{h}\bm{X})^{-1}\bm{X}^{\top}(\bm{W}_{h}-I_{n})\bm{H}\bm{W}_{g}\bm{e}_{g}\\\
&+(\lambda p)^{-1}\sum_{h\neq
g}\bar{\bm{\ell}}_{hr}\bm{e}_{h}^{\top}\bm{W}_{h}\bm{X}(\bm{X}^{\top}\bm{W}_{h}\bm{X})^{-1}\bm{X}^{\top}(\bm{W}_{h}-I_{n})\bm{H}(\bm{W}_{g}-I_{n})\bm{X}(\bm{X}^{\top}\bm{W}_{g}\bm{X})^{-1}\\\
&\times\bm{X}^{\top}\bm{W}_{g}\bm{e}_{g}.\end{aligned}$ (S8.32)

First, $\lVert n^{-1/2}\bm{e}_{h}^{\top}\bm{W}_{h}\bm{Z}\rVert_{2}\leq\lVert
n^{-1/2}\bm{e}_{h}^{\top}\bm{Z}\rVert_{2}+\lVert
n^{-1/2}\bm{e}_{h}^{\top}(\bm{W}_{h}-I_{n})\bm{Z}\rVert_{2}$, where
$\max_{g\in[p]}\lVert
n^{-1/2}\bm{e}_{h}^{\top}\bm{Z}\rVert_{2}=O_{P}(n^{\epsilon})$ by Lemma S8.2
and Corollary S8.1 implies $\max_{g\in[p]}\lVert
n^{-1/2}\bm{e}_{h}^{\top}(\bm{W}_{h}-I_{n})\bm{Z}\rVert_{2}=O_{P}(n^{\epsilon})$
for any $\epsilon>0$. Therefore, the second term in (S8.32) satisfies

$\displaystyle\max_{g\in[p]}\lvert(\lambda p)^{-1}\sum_{h\neq