$\\{Y^{L,\perp}_{t,h},Y^{R,\perp}_{t,h}\in\mathbb{R}^{r_{h}}:h\in[m]\\}$
generated as follows:

$\displaystyle Y^{L,\perp}_{t,h}$ $\displaystyle\sim
N_{r_{h}}\bigg{(}\bm{A}^{L,\perp}_{t,h}U_{h}+\bm{B}^{L,\perp}_{t,h}Y_{0,h},\bm{T}^{L,\perp}_{t,h}\bigg{)}$
(C.17) $\displaystyle Y^{R,\perp}_{t,h}$ $\displaystyle\sim
N_{r_{h}}\bigg{(}\bm{A}^{R,\perp}_{t,h}V_{h}+\bm{B}^{R,\perp}_{t,h}Y^{R}_{0,h},\bm{T}^{R,\perp}_{t,h}\bigg{)},$
(C.18)

In the above equation,
$(U_{1},\ldots,U_{m},\widetilde{U}_{1},\ldots,\widetilde{U}_{\widetilde{m}})\sim\mu$,
$V_{h}\sim\nu_{h}$ for all $h\in[m]$ and $\\{Y_{0,h},Y^{R}_{0,h}:h\in[m]\\}$
are defined in (2.12) and (2.15) respectively. Furthermore, for all $h\in[m]$,
the matrices

$\bm{A}^{L,\perp}_{t,h},\bm{B}^{L,\perp}_{t,h},\bm{T}^{L,\perp}_{t,h},\bm{A}^{R,\perp}_{t,h},\bm{B}^{R,\perp}_{t,h},\bm{T}^{R,\perp}_{t,h}\in\mathbb{R}^{r_{h}\times
r_{h}}$

are recursively defined as follows:

$\displaystyle\bm{A}^{R,\perp}_{0,h}=\bm{0},\quad\bm{B}^{R,\perp}_{0,h}=\bm{I}_{r_{h}},\quad\bm{T}^{R,\perp}_{0,h}=\bm{0},\quad\bm{A}^{L,\perp}_{-1,h}=\bm{0},\quad\bm{B}^{L,\perp}_{-1,h}=(\bm{\Sigma}^{L,\mathrm{pc}}_{0,h})^{1/2},\quad\bm{T}^{L,\perp}_{-1,h}=\bm{0},$
(C.19) $\displaystyle\bm{A}^{L,\perp}_{t,h}$
$\displaystyle=\gamma_{h}\bigg{\\{}\mathbb{E}_{\nu_{h}}[v^{\circ}_{t,h}(Y^{R,\perp}_{t,h};\nu_{h})V^{\top}_{h}]\bm{D}_{h}-\mathbb{E}_{\nu_{h}}[v^{\circ}_{t,h}(Y^{R,\perp}_{t,h};\nu_{h})(Y^{R}_{0,h})^{\top}]\bm{S}^{R,\mathrm{pc}}_{0,h}\bm{D}_{h}\bigg{\\}},$
(C.20) $\displaystyle\bm{B}^{L,\perp}_{t,h}$
$\displaystyle=\gamma_{h}\bigg{\\{}\mathbb{E}_{\nu_{h}}[v^{\circ}_{t,h}(Y^{R,\perp}_{t,h};\nu_{h})(Y^{R}_{0,h})^{\top}]\bm{D}^{\ast}_{h}-\mathbb{E}_{\nu_{h}}[v^{\circ}_{t,h}(Y^{R,\perp}_{t,h};\nu_{h})V^{\top}_{h}]\bm{D}_{h}(\bm{S}^{L,\mathrm{pc}}_{0,h})^{\top}$
(C.21) $\displaystyle\hskip
20.00003pt+\mathbb{E}_{\nu_{h}}[v^{\circ}_{t,h}(Y^{R,\perp}_{t,h};\nu_{h})(Y^{R}_{0,h})^{\top}]\bm{S}^{R,\mathrm{pc}}_{0,h}\bm{D}_{h}(\bm{S}^{L,\mathrm{pc}}_{0,h})^{\top}$
(C.22) $\displaystyle\hskip
20.00003pt-\mathbb{E}_{\nu_{h}}[\mathsf{d}v^{\circ}_{t,h}(Y^{R,\perp}_{t,h};\nu_{h})]\mathbb{E}_{\mu}[u^{\circ}_{t-1,h}(Y^{L,\perp}_{t-1,h},\ldots,Y^{L,\perp}_{t-1,m},\widetilde{Y}_{0,1},\ldots,\widetilde{Y}_{0,\widetilde{m}})(Y_{0,h})^{\top}]\bigg{\\}},$
(C.23) $\displaystyle\bm{T}^{L,\perp}_{t,h}$
$\displaystyle=\gamma_{h}\bigg{\\{}\mathbb{E}_{\nu_{h}}[v^{\circ}_{t,h}(Y^{R,\perp}_{t,h};\nu_{h})v^{\circ}_{t,h}(Y^{R,\perp}_{t,h};\nu_{h})^{\top}]$
(C.24) $\displaystyle\hskip
20.00003pt-\mathbb{E}_{\nu_{h}}[v_{t,h}(Y^{R,\perp}_{t,h};\nu_{h})(Y^{R}_{0,h})^{\top}]\mathbb{E}_{\nu_{h}}[(Y^{R}_{0,h})v_{t,h}(Y^{R,\perp}_{t,h};\nu_{h})^{\top}]\bigg{\\}},$
(C.25) $\displaystyle\bm{A}^{R,\perp}_{t+1,h}$
$\displaystyle=\mathbb{E}_{\mu}[u^{\circ}_{t,h}(Y^{L,\perp}_{t,1},\ldots,Y^{L,\perp}_{t,m},\widetilde{Y}_{0,1},\ldots,\widetilde{Y}_{0,\widetilde{m}};\mu)U^{\top}_{h}]\bm{D}_{h}$
(C.26) $\displaystyle\hskip
20.00003pt-\mathbb{E}[u^{\circ}_{t,h}(Y^{L,\perp}_{t,1},\ldots,Y^{L,\perp}_{t,m},\widetilde{Y}_{0,1},\ldots,\widetilde{Y}_{0,\widetilde{m}};\mu)(Y_{0,h})^{\top}]\bm{S}^{L,\mathrm{pc}}_{0,h}\bm{D}_{h},$
(C.27) $\displaystyle\bm{B}^{R,\perp}_{t+1,h}$
$\displaystyle=\mathbb{E}_{\mu}[u^{\circ}_{t,h}(Y^{L,\perp}_{t,1},\ldots,Y^{L,\perp}_{t,m},\widetilde{Y}_{0,1},\ldots,\widetilde{Y}_{0,\widetilde{m}};\mu)Y^{\top}_{0,h}]\bm{D}^{*}_{h}$
(C.28) $\displaystyle\hskip
20.00003pt-\mathbb{E}_{\mu}[u^{\circ}_{t,h}(Y^{L,\perp}_{t,1},\ldots,Y^{L,\perp}_{t,m},\widetilde{Y}_{0,1},\ldots,\widetilde{Y}_{0,\widetilde{m}};\mu)U^{\top}_{h}]\bm{D}_{h}(\bm{S}^{R,\mathrm{pc}}_{0,h})^{\top}$
(C.29) $\displaystyle\hskip
20.00003pt+\mathbb{E}_{\mu}[u^{\circ}_{t,h}(Y^{L,\perp}_{t,1},\ldots,Y^{L,\perp}_{t,m},\widetilde{Y}_{0,1},\ldots,\widetilde{Y}_{0,\widetilde{m}};\mu)Y^{\top}_{0,h}]\bm{S}^{L,\mathrm{pc}}_{0,h}\bm{D}_{h}(\bm{S}^{R,\mathrm{pc}}_{0,h})^{\top}$
(C.30) $\displaystyle\hskip
20.00003pt-\mathbb{E}_{\mu}[\mathsf{d}_{h}u^{\circ}_{t,h}(Y^{L,\perp}_{t,1},\ldots,Y^{L,\perp}_{t,m},\widetilde{Y}_{0,1},\ldots,\widetilde{Y}_{0,\widetilde{m}};\mu)]\mathbb{E}_{\nu_{h}}[v^{\circ}_{t,h}(Y^{R,\perp}_{t,h})(Y^{R}_{0,h})^{\top}],$
(C.31) $\displaystyle\bm{T}^{R,\perp}_{t+1,h}$
$\displaystyle=\mathbb{E}_{\mu}[u^{\circ}_{t,h}(Y^{L,\perp}_{t,1},\ldots,Y^{L,\perp}_{t,m},\widetilde{Y}_{0,1},\ldots,\widetilde{Y}_{0,\widetilde{m}};\mu)^{\otimes
2}]$ (C.32) $\displaystyle\hskip
20.00003pt-\mathbb{E}_{\mu}[u^{\circ}_{t,h}(Y^{L,\perp}_{t,1},\ldots,Y^{L,\perp}_{t,m},\widetilde{Y}_{0,1},\ldots,\widetilde{Y}_{0,\widetilde{m}};\mu)Y^{\top}_{0,h}]^{\otimes
2}$ (C.33)

where $\lim_{N\rightarrow\infty}\bm{D}_{0,h}\overset{a.s.}{=}\bm{D}^{*}_{h}$,
the matrices $\\{\mathsf{d}v^{\circ}_{t,h}(\cdot):h\in[m]\\}$ are the Jacobian
matrices of the functions $\\{v^{\circ}_{t,h}(\cdot):h\in[m]\\}$ and
$\\{\mathsf{d}_{h}u^{\circ}_{t,h}(\cdot):h\in[m]\\}$ are the Jacobian matrices
of the function $\\{u^{\circ}_{t,h}(\cdot):h\in[m]\\}$ with respect to the
variables corresponding to the $h$-th modality. By convention we set

$\mathbb{E}[u^{\circ}_{-1,h}(Y^{L,\perp}_{-1,1},\ldots,Y^{L,\perp}_{-1,m},\widetilde{Y}_{0,1},\ldots,\widetilde{Y}_{0,\widetilde{m}};\mu)Y^{\top}_{0,h}]=(\bm{\Sigma}^{L,\mathrm{pc}}_{0,h})^{1/2}.$

Next, consider the following lemmas.

######  Lemma C.1.

Consider the sequence of deterministic subsets $\mathcal{F}_{L}\subset[N]$
satisfying $\frac{|\mathcal{F}_{L}|}{N}\rightarrow\lambda_{L}\in(0,1]$, as
$N\rightarrow\infty$ and $\\{\mathcal{F}_{R,h}\subset[p_{h}]:h\in[m]\\}$
satisfying
$\frac{|\mathcal{F}_{R,h}|}{p_{h}}\rightarrow\lambda_{R,h}\in(0,1]$, as
$p_{h}\rightarrow\infty$ for all $h\in[m]$. Then, for any pseudo-Lipschitz
functions $\psi:\mathbb{R}^{3r+2\widetilde{r}}\rightarrow\mathbb{R}$ and
$\phi_{h}:\mathbb{R}^{3r_{h}}\rightarrow\mathbb{R}$ for $h\in[m]$, the
following limits hold almost surely.

$\displaystyle\lim_{N\rightarrow\infty}\Bigg{|}\frac{1}{|\mathcal{F}_{L}|}\sum_{i\in\mathcal{F}_{L}}\psi\left(\\{(\bm{U}^{\perp}_{t,h})_{i*}\\}_{h=1}^{m},\\{({\bm{U}}^{\mathrm{pc}}_{0,h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{X}}_{h})_{i*}\\}_{h=1}^{\widetilde{m}},\\{(\bm{U}_{h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{U}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}}\right)$
(C.34) $\displaystyle\hskip
10.00002pt-\mathbb{E}_{\mu}\left[\psi\left(\\{\bm{A}^{L,\perp}_{t,h}U_{h}+\bm{B}^{L,\perp}_{t,h}Y_{0,h}+(\bm{T}^{L,\perp}_{t,h})^{1/2}W^{\perp,L}_{t,h}\\}_{h=1}^{m},\\{Y_{0,h}\\}_{h=1}^{m},\\{\widetilde{Y}_{0,\ell}\\}_{\ell=1}^{\widetilde{m}},\\{U_{h}\\}_{h=1}^{m},\\{\widetilde{U}_{\ell}\\}_{\ell=1}^{\widetilde{m}}\right)\right]\Bigg{|}$
(C.35) $\displaystyle\hskip 20.00003pt=0,$ (C.36)

and

$\displaystyle\lim_{p_{h}\rightarrow\infty}\Bigg{|}\frac{1}{|\mathcal{F}_{R,h}|}\sum_{i\in\mathcal{F}_{R,h}}\phi_{h}\left((\bm{V}^{\perp}_{t,h})_{i*},({\bm{V}}^{\mathrm{pc}}_{0,h})_{i*},(\bm{V}_{h})_{i*}\right)$
(C.37) $\displaystyle\hskip
40.00006pt-\mathbb{E}_{\nu_{h}}\left[\phi_{h}\left(\bm{A}^{R,\perp}_{t,h}V_{h}+\bm{B}^{R,\perp}_{t,h}Y^{R}_{0,h}+(\bm{T}^{R,\perp}_{t,h})^{1/2}W^{\perp,R}_{t,h},Y^{R}_{0,h},V_{h}\right)\right]\Bigg{|}=0,$
(C.38)

where for all $t\geq 0$ and $h\in[m]$, the random variables
$W^{\perp,L}_{t,h},W^{\perp,R}_{t,h}\sim N_{r_{h}}(0,\bm{I}_{r_{h}})$ and are
mutually independent. In addition,
$(U_{1},\ldots,U_{m},\widetilde{U}_{1},\ldots,\widetilde{U}_{\widetilde{m}})\sim\mu$
and $\\{V_{h}\sim\nu_{h}:h\in[m]\\}$.

######  Lemma C.2.

For any sequence of pseudo-Lipschitz functions
$\psi:\mathbb{R}^{3r+2\widetilde{r}}\rightarrow\mathbb{R}$ and
$\phi_{h}:\mathbb{R}^{3r_{h}}\rightarrow\mathbb{R}$ for $h\in[m]$, the
following limits hold almost surely.

$\displaystyle\lim_{N\rightarrow\infty}\Bigg{|}\mathbb{E}_{\mu}\left[\psi\left(\\{\bm{A}^{L,\perp}_{t,h}U_{h}+\bm{B}^{L,\perp}_{t,h}Y_{0,h}+(\bm{T}^{L,\perp}_{t,h})^{1/2}W^{\perp,L}_{t,h}\\}_{h=1}^{m},\\{Y_{0,h}\\}_{h=1}^{m},\\{\widetilde{Y}_{0,\ell}\\}_{\ell=1}^{\widetilde{m}},\\{U_{h}\\}_{h=1}^{m},\\{\widetilde{U}_{\ell}\\}_{\ell=1}^{\widetilde{m}}\right)\right]$
(C.39) $\displaystyle\hskip
40.00006pt-\mathbb{E}_{\mu}\left[\psi\left(\\{\bm{S}^{L,\circ}_{t,h}U_{h}+(\bm{\Sigma}^{L,\circ}_{t,h})^{1/2}Z^{\perp,L}_{t,h}\\}_{h=1}^{m},\\{Y_{0,h}\\}_{h=1}^{m},\\{\widetilde{Y}_{0,\ell}\\}_{\ell=1}^{\widetilde{m}},\\{U_{h}\\}_{h=1}^{m},\\{\widetilde{U}_{\ell}\\}_{\ell=1}^{\widetilde{m}}\right)\right]\Bigg{|}=0,$
(C.40)

and

$\displaystyle\lim_{p_{h}\rightarrow\infty}\Bigg{|}\mathbb{E}_{\nu_{h}}\left[\phi_{h}\left(\bm{A}^{R,\perp}_{t,h}V_{h}+\bm{B}^{R,\perp}_{t,h}Y^{R}_{0,h}+(\bm{T}^{R,\perp}_{t,h})^{1/2}W^{\perp,R}_{t,h},Y^{R}_{0,h},V_{h}\right)\right]-$
(C.41) $\displaystyle\hskip
40.00006pt\mathbb{E}_{\nu_{h}}\left[\phi_{h}\left(\bm{S}^{R,\circ}_{t,h}V_{h}+(\bm{\Sigma}^{R,\circ}_{t,h})^{1/2}Z^{\perp,R}_{t,h},Y^{R}_{0,h},V_{h}\right)\right]\Bigg{|}=0,$
(C.42)

where for all $t\geq 0$ and $h\in[m]$, the random variables
$Z^{\perp,L}_{t,h},Z^{\perp,R}_{t,h}\sim N_{r_{h}}(0,\bm{I}_{r_{h}})$ and are
independent.

###### Proof of Theorem C.1.

Using (C.14) and (C.16) along with Lemmas C.1 and C.2, the result follows. ∎

###  C.2 Proof of Lemma C.2

Observe that by definition, as $N\rightarrow\infty$

$\|\bm{S}^{R,\mathrm{pc}}_{0,h}\bm{D}_{h}-\bm{D}_{h}\bm{S}^{R,\mathrm{pc}}_{0,h}\|^{2}_{F}\xrightarrow{a.s.}0,\quad\mbox{and}\quad\|\bm{S}^{L,\mathrm{pc}}_{0,h}\bm{D}_{h}-\bm{D}_{h}\bm{S}^{L,\mathrm{pc}}_{0,h}\|^{2}_{F}\xrightarrow{a.s.}0.$

Using the definitions of
$\bm{S}^{L,\mathrm{pc}}_{0,h},\bm{S}^{R,\mathrm{pc}}_{0,h}$ from (2.7) and
that of $\\{\bm{B}^{L,\perp}_{t,h},\bm{B}^{R,\perp}_{t,h}\\}_{t\geq-1}$ we get
the following limits using the Stein’s Lemma [40].

$\displaystyle\lim_{N\rightarrow\infty}\|\bm{B}^{R,\perp}_{t+1,h}\bm{S}^{R,\mathrm{pc}}_{0,h}-\mathbb{E}_{\mu}[u^{\circ}_{t,h}(Y^{L,\perp}_{t,1},\ldots,Y^{L,\perp}_{t,m},\widetilde{Y}_{0,1},\ldots,\widetilde{Y}_{0,\widetilde{m}};\mu)Y^{\top}_{0,h}]\bm{S}^{L,\mathrm{pc}}_{0,h}\bm{D}_{h}\|^{2}_{F}\overset{a.s.}{=}0,$
(C.43)
$\displaystyle\lim_{p_{h}\rightarrow\infty}\|\bm{B}^{L,\perp}_{t+1,h}\bm{S}^{L,\mathrm{pc}}_{0,h}-\gamma_{h}\mathbb{E}_{\nu_{h}}[v^{\circ}_{t,h}(Y^{R,\perp}_{t,h};\nu_{h})(Y^{R}_{0,h})^{\top}]\bm{S}^{R,\mathrm{pc}}_{0,h}\bm{D}_{h}\|^{2}_{F}\overset{a.s.}{=}0.$
(C.44)

For $h\in[m]$, let us define:

$\displaystyle\mathtt{S}^{L}_{t,h}(\\{\bm{M}_{h}\\}_{h=1}^{m},\\{\bm{Q}_{h}\\}_{h=1}^{m})$
$\displaystyle\equiv\mathbb{E}_{\mu}\bigg{[}u^{\circ}_{t,h}\bigg{(}\\{\bm{M}_{h}U_{h}+\bm{Q}^{1/2}_{h}Z_{h}\\}_{h=1}^{m},\\{\bm{L}_{\ell}\widetilde{U}_{\ell}+\widetilde{Z}_{\ell}\\}_{\ell=1}^{\widetilde{m}};\mu\bigg{)}U^{\top}_{h}\bigg{]}\bm{D}_{h},$
(C.45)
$\displaystyle\mathtt{\Sigma}^{L}_{t,h}(\\{\bm{M}_{h}\\}_{h=1}^{m},\\{\bm{Q}_{h}\\}_{h=1}^{m})$
$\displaystyle\equiv\mathbb{E}_{\mu}\bigg{[}u^{\circ}_{t,h}\bigg{(}\\{\bm{M}_{h}U_{h}+\bm{Q}^{1/2}_{h}Z_{h}\\}_{h=1}^{m},\\{\bm{L}_{\ell}\widetilde{U}_{\ell}+\widetilde{Z}_{\ell}\\}_{\ell=1}^{\widetilde{m}};\mu\bigg{)}^{\otimes
2}\bigg{]},$ (C.46) $\displaystyle\mathtt{S}^{R}_{t,h}(\bm{M}_{h},\bm{Q}_{h})$
$\displaystyle\equiv\gamma_{h}\mathbb{E}_{\nu_{h}}[v^{\circ}_{t,h}(\bm{M}_{h}V_{h}+\bm{Q}^{1/2}_{h}Z_{h};\nu_{h})V^{\top}_{h}]\bm{D}_{h}$
(C.47) $\displaystyle\mathtt{\Sigma}^{R}_{t,h}(\bm{M}_{h},\bm{Q}_{h})$
$\displaystyle\equiv\gamma_{h}\mathbb{E}_{\nu_{h}}[v^{\circ}_{t,h}(\bm{M}_{h}V_{h}+\bm{Q}^{1/2}_{h}Z_{h};\nu_{h})^{\otimes
2}].$ (C.48)

Let us define for $h\in[m]$, the parameters
$\bm{S}^{L,\perp}_{t,h}\equiv\bm{A}^{L,\perp}_{t,h}+\bm{B}^{L,\perp}_{t,h}\bm{S}^{L,\mathrm{pc}}_{0,h}$
and
$\bm{\Sigma}^{L,\perp}_{t,h}\equiv\bm{T}^{L,\perp}_{t,h}+\bm{B}^{L,\perp}_{t,h}\bm{\Sigma}^{L,\mathrm{pc}}_{0,h}(\bm{B}^{L,\perp}_{t,h})^{\top}$.
By construction, it is evident that for all $h\in[m]$ we have:

$\displaystyle\mathbb{E}_{\mu}\left[\psi\left(\\{\bm{A}^{L,\perp}_{t,h}U_{h}+\bm{B}^{L,\perp}_{t,h}Y_{0,h}+(\bm{T}^{L,\perp}_{t,h})^{1/2}W^{\perp,L}_{t,h}\\}_{h=1}^{m},\\{Y_{0,h}\\}_{h=1}^{m},\\{\widetilde{Y}_{0,\ell}\\}_{\ell=1}^{\widetilde{m}},\\{U_{h}\\}_{h=1}^{m},\\{\widetilde{U}_{\ell}\\}_{\ell=1}^{\widetilde{m}}\right)\right]$
(C.49) $\displaystyle\hskip
40.00006pt=\mathbb{E}_{\mu}\left[\psi\left(\\{\bm{S}^{L,\perp}_{t,h}U_{h}+(\bm{\Sigma}^{L,\perp}_{t,h})^{1/2}Z^{\perp,L}_{t,h}\\}_{h=1}^{m},\\{Y_{0,h}\\}_{h=1}^{m},\\{\widetilde{Y}_{0,\ell}\\}_{\ell=1}^{\widetilde{m}},\\{U_{h}\\}_{h=1}^{m},\\{\widetilde{U}_{\ell}\\}_{\ell=1}^{\widetilde{m}}\right)\right],$
(C.50)

From the definition of the functions $\mathtt{S}^{R}_{t,h}$ and
$\mathtt{\Sigma}^{R}_{t,h}$, using (C.43) we can show that

$\displaystyle\lim_{N\rightarrow\infty}\|\bm{S}^{L,\perp}_{t,h}-\mathtt{S}^{R}_{t,h}(\bm{S}^{R,\perp}_{t,h},\bm{\Sigma}^{R,\perp}_{t,h})\|_{F}=0,$
(C.51)
$\displaystyle\lim_{N\rightarrow\infty}\|\bm{\Sigma}^{L,\perp}_{t,h}-\mathtt{\Sigma}^{R}_{t,h}(\bm{S}^{R,\perp}_{t,h},\bm{\Sigma}^{R,\perp}_{t,h})\|_{F}=0.$
(C.52)

Similarly, we can show that

$\displaystyle\lim_{N\rightarrow\infty}\|\bm{S}^{R,\perp}_{t+1,h}-\mathtt{S}^{L}_{t,h}(\\{\bm{S}^{L,\perp}_{t,h}\\}_{h=1}^{m},\\{\bm{\Sigma}^{L,\perp}_{t,h}\\}_{h=1}^{m})\|_{F}=0,$
(C.53)
$\displaystyle\lim_{N\rightarrow\infty}\|\bm{\Sigma}^{R,\perp}_{t,h}-\mathtt{\Sigma}^{L}_{t,h}(\\{\bm{S}^{L,\perp}_{t,h}\\}_{h=1}^{m},\\{\bm{\Sigma}^{L,\perp}_{t,h}\\}_{h=1}^{m})\|_{F}=0.$
(C.54)

Finally, using the definition of the state evolution parameters (4.9), and
$\bm{S}^{R,\perp}_{0,h}=\bm{S}^{R,\circ}_{0,h}$,
$\bm{\Sigma}^{R,\perp}_{0,h}=\bm{\Sigma}^{R,\circ}_{0,h}$ for all $h\in[m]$,
using continuity of the functions $\mathtt{S}^{R}_{t,h}$ and
$\mathtt{\Sigma}^{R}_{t,h}$ we can inductively conclude that

$\lim_{N\rightarrow\infty}\|\bm{S}^{L,\perp}_{t,h}-\bm{S}^{L,\circ}_{t,h}\|_{F}=0\quad\mbox{and}\quad\lim_{N\rightarrow\infty}\|\bm{\Sigma}^{L,\perp}_{t,h}-\bm{\Sigma}^{L,\circ}_{t,h}\|_{F}=0.$

Hence, the first assertion follows using the pseudo-Lipschitz property of
$\psi$. The second assertion also follows using similar arguments.

###  C.3 Proof of Lemma C.1

Let us consider the functions

$\displaystyle
u^{\perp,\circ}_{t,h}(x_{1},\ldots,x_{m},\widetilde{x}_{1},\ldots,\widetilde{x}_{\widetilde{m}};\mu)$
(C.55)
$\displaystyle=u^{\circ}_{t,h}(x_{1},\ldots,x_{m},\widetilde{x}_{1},\ldots,\widetilde{x}_{\widetilde{m}};\mu)$
(C.56) $\displaystyle\hskip
15.00002pt-{\bm{U}}^{\mathrm{pc}}_{0,h}\,\mathbb{E}_{\mu}[Y_{0,h}u^{\circ}_{t,h}(\\{\bm{A}^{L,\perp}_{t,h}U_{h}+\bm{B}^{L,\perp}_{t,h}Y_{0,h}+(\bm{T}^{L,\perp}_{t,h})^{1/2}W^{\perp,L}_{t,h}\\}_{h=1}^{m},\\{\widetilde{Y}_{0,\ell}\\}_{\ell=1}^{\widetilde{m}})^{\top}],$
(C.57)

and

$\displaystyle
v^{\perp,\circ}_{t,h}(y;\nu_{h})=v^{\circ}_{t,h}(y;\nu_{h})-{\bm{V}}^{\mathrm{pc}}_{0,h}\,\mathbb{E}_{\nu_{h}}[Y^{R}_{0,h}v^{\circ}_{t,h}(\bm{A}^{R,\perp}_{t,h}V_{h}+\bm{B}^{R,\perp}_{t,h}Y^{R}_{0,h}+(\bm{T}^{R,\perp}_{t,h})^{1/2}W^{\perp,R}_{t,h})^{\top}].$
(C.58)

For all $h\in[m]$, let us define the sequence of auxiliary iterations
$\\{\widebar{\bm{N}}^{\perp,L}_{t,h},\widebar{\bm{N}}^{\perp,R}_{t,h}\\}_{t\geq
0}$ as follows:

$\displaystyle\widebar{\bm{N}}^{\perp,R}_{t,h}=v^{\perp,\circ}_{t,h}({\bm{N}}^{\perp,R}_{t,h}+\bm{V}_{h}(\bm{A}^{R,\perp}_{t,h})^{\top}+{\bm{V}}^{\mathrm{pc}}_{0,h}(\bm{B}^{R,\perp}_{t,h})^{\top};\nu_{h}),$
(C.59)
$\displaystyle{\bm{N}}^{\perp,L}_{t,h}=\bm{W}^{\perp}_{h}\widebar{\bm{N}}^{\perp,R}_{t,h}-\widebar{\bm{N}}^{\perp,L}_{t-1,h}\,\gamma_{h}\left(\bm{J}^{R,\perp}_{t,h}({\bm{N}}^{\perp,R}_{t,h}+\bm{V}_{h}(\bm{A}^{R,\perp}_{t,h})^{\top}+{\bm{V}}^{\mathrm{pc}}_{0,h}(\bm{B}^{R,\perp}_{t,h})^{\top};\nu_{h})\right)^{\top},$
(C.60)
$\displaystyle\widebar{\bm{N}}^{\perp,L}_{t,h}=u^{\perp,\circ}_{t,h}({\bm{N}}^{\perp,L}_{t,1}+\bm{U}_{1}(\bm{A}^{L,\perp}_{t,1})^{\top}+{\bm{U}}^{\mathrm{pc}}_{0,1}(\bm{B}^{L,\perp}_{t,1})^{\top},\ldots,{\bm{N}}^{\perp,L}_{t,m}+\bm{U}_{m}(\bm{A}^{L,\perp}_{t,m})^{\top}+{\bm{U}}^{\mathrm{pc}}_{0,m}(\bm{B}^{L,\perp}_{t,m})^{\top},$
(C.61) $\displaystyle\hskip
350.00053pt\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}};\mu),$
(C.62)
$\displaystyle{\bm{N}}^{\perp,R}_{t+1,h}=(\bm{W}^{\perp}_{h})^{\top}\widebar{\bm{N}}^{\perp,L}_{t,h}-\widebar{\bm{N}}^{\perp,R}_{t,h}\,\bigg{(}\bm{J}^{L,\perp}_{t,h}({\bm{N}}^{\perp,L}_{t,1}+\bm{U}_{1}(\bm{A}^{L,\perp}_{t,1})^{\top}+{\bm{U}}^{\mathrm{pc}}_{0,1}(\bm{B}^{L,\perp}_{t,1})^{\top},\ldots,$
(C.63) $\displaystyle\hskip
180.00027pt{\bm{N}}^{\perp,L}_{t,m}+\bm{U}_{m}(\bm{A}^{L,\perp}_{t,m})^{\top}+{\bm{U}}^{\mathrm{pc}}_{0,m}(\bm{B}^{L,\perp}_{t,m})^{\top},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}};\mu)\bigg{)}^{\top}.$
(C.64)

In the above definitions $\\{\bm{W}^{\perp}_{h}\in\mathbb{R}^{N\times
p_{h}}:(\bm{W}^{\perp}_{h})_{ij}\overset{iid}{\sim}N(0,N^{-1}),h\in[m]\\}$ and
is independent of the observed data. Further, the matrix-valued functions
$\bm{J}^{R,\perp}_{t,h}(\cdot):\mathbb{R}^{p_{h}\times
r_{h}}\rightarrow\mathbb{R}^{r_{h}\times r_{h}}$ is defined similarly as
(4.14) with $v^{\perp,\circ}_{t,h}(\cdot)$ replacing $v^{\circ}_{t,h}(\cdot)$.
Similarly, the matrix-valued functions
$\bm{J}^{L,\perp}_{t,h}(\cdot):\prod_{k=1}^{m}\mathbb{R}^{N\times
r_{k}}\times\prod_{\ell=1}^{\widetilde{m}}\mathbb{R}^{N\times{\widetilde{r}}_{\ell}}\rightarrow\mathbb{R}^{r_{h}\times
r_{h}}$ is defined similarly as (4.15) with $u^{\perp,\circ}_{t,h}(\cdot)$
replacing $u^{\circ}_{t,h}(\cdot)$. We initialize the iterations (C.59) as
follows:

$\displaystyle{\bm{N}}^{\perp,R}_{0,h}=\bm{V}^{\perp}_{0,h}-\bm{V}_{h}(\bm{A}^{R,\perp}_{0,h})^{\top}-{\bm{V}}^{\mathrm{pc}}_{0,h}(\bm{B}^{R,\perp}_{0,h})^{\top}=0,$
(C.65)
$\displaystyle{\bm{N}}^{\perp,L}_{-1,h}=\bm{U}^{\perp}_{-1,h}-\bm{U}_{h}(\bm{A}^{L,\perp}_{-1,h})^{\top}-{\bm{U}}^{\mathrm{pc}}_{0,h}(\bm{B}^{L,\perp}_{-1,h})^{\top}=0,$
(C.66)

for all $h\in[m]$. Now we consider the following lemma.

######  Lemma C.3.

Let
$u^{\mathtt{g}}_{t,h}:\mathbb{R}^{r+\widetilde{r}}\rightarrow\mathbb{R}^{r_{h}}$,
$v^{\mathtt{g}}_{t,h}:\mathbb{R}^{r_{h}}\rightarrow\mathbb{R}^{r_{h}}$ be
Lipschitz functions with Lipschitz Jacobian matrices. Let
$\bm{G}_{h}\in\mathbb{R}^{N\times p_{h}}$ for $h\in[m]$ be random matrices
with iid Gaussian entries with zero mean and variance $N^{-1}$. The matrices
$\bm{G}_{h}$ are independent of one another for $h\in[m]$. Consider the
sequence of AMP iterates defined as follows:

$\displaystyle\widebar{\bm{N}}^{G,R}_{t,h}=v^{\mathtt{g}}_{t,h}({\bm{N}}^{G,R}_{t,h}),$
(C.67)
$\displaystyle{\bm{N}}^{G,L}_{t,h}=\bm{G}_{h}\widebar{\bm{N}}^{G,R}_{t,h}-\widebar{\bm{N}}^{G,L}_{t-1,h}\,\gamma_{h}\left(J^{\mathtt{g,R}}_{t,h}({\bm{N}}^{G,R}_{t,h})\right)^{\top},$
(C.68)
$\displaystyle\widebar{\bm{N}}^{G,L}_{t,h}=u^{\mathtt{g}}_{t,h}({\bm{N}}^{G,L}_{t,1},\ldots,{\bm{N}}^{G,L}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}}),$
(C.69)
$\displaystyle{\bm{N}}^{G,R}_{t+1,h}=\bm{G}^{\top}_{h}\widebar{\bm{N}}^{G,L}_{t,h}-\widebar{\bm{N}}^{G,R}_{t,h}\,\left(J^{\mathtt{g,L}}_{t,h}({\bm{N}}^{G,L}_{t,1},\ldots,{\bm{N}}^{G,L}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}})\right)^{\top}.$
(C.70)

where ${\bm{N}}^{G,R}_{0,h}=\bm{0}$ and
$\widebar{\bm{N}}^{G,L}_{-1,h}=\bm{0}$. In the above definitions, the matrix-
valued functions $J^{\mathtt{g,R}}_{t,h}:\mathbb{R}^{p_{h}\times
r_{h}}\rightarrow\mathbb{R}^{r_{h}\times r_{h}}$ corresponding to the function
$v^{\mathtt{g}}_{t,h}(\bm{x})$ is defined as follows:

$J^{\mathtt{g,R}}_{t,h}({\bm{N}}^{G,R}_{t,h})=\frac{1}{p_{h}}\sum_{i=1}^{p_{h}}\frac{\bm{\partial}v^{\mathtt{g}}_{t,h}}{\bm{\partial}\bm{x}}(({\bm{N}}^{G,R}_{t,h})_{i*})$

Similarly, the matrix-valued functions
$J^{\mathtt{g,L}}_{t,h}:\prod_{k=1}^{m}\mathbb{R}^{N\times
r_{k}}\times\prod_{\ell=1}^{\widetilde{m}}\mathbb{R}^{N\times{\widetilde{r}}_{\ell}}\rightarrow\mathbb{R}^{r_{h}\times
r_{h}}$ corresponding to the function
$u^{\mathtt{g}}_{t,h}(\bm{x}_{1},\ldots,\bm{x}_{m},\widetilde{\bm{x}}_{1},\ldots,\widetilde{\bm{x}}_{\widetilde{m}})$
is defined as follows:

$\displaystyle
J^{\mathtt{g,L}}_{t,h}({\bm{N}}^{G,L}_{t,1},\ldots,{\bm{N}}^{G,L}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}})$
(C.71)
$\displaystyle\quad=\frac{1}{N}\sum_{i=1}^{N}\frac{\bm{\partial}u^{\mathtt{g}}_{t,h}}{\bm{\partial}\bm{x}_{h}}(({\bm{N}}^{G,L}_{t,1})_{i*},\ldots,({\bm{N}}^{G,L}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*}).$
(C.72)

Next, consider any sequence of deterministic subsets
$\mathcal{F}_{L}\subset[N]$ satisfying
$\frac{|\mathcal{F}_{L}|}{N}\rightarrow\lambda_{L}\in(0,1]$, as
$N\rightarrow\infty$ and $\\{\mathcal{F}_{R,h}\subset[p_{h}]:h\in[m]\\}$
satisfying
$\frac{|\mathcal{F}_{R,h}|}{p_{h}}\rightarrow\lambda_{R,h}\in(0,1]$, as
$p_{h}\rightarrow\infty$ for all $h\in[m]$. Then, for any pseudo Lipschitz
functions $\psi:\mathbb{R}^{3r+2\widetilde{r}}\rightarrow\mathbb{R}$ and
$\phi_{h}:\mathbb{R}^{3r_{h}}\rightarrow\mathbb{R}$ we have:

$\displaystyle\lim_{N\rightarrow\infty}\Bigg{|}\frac{1}{|\mathcal{F}_{L}|}\sum_{i\in\mathcal{F}_{L}}\psi\left(\\{({\bm{N}}^{G,L}_{t,h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{X}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}},\\{({\bm{U}}^{\mathrm{pc}}_{0,h})_{i*}\\}_{h=1}^{m},\\{(\bm{U}_{h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{U}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}}\right)$
(C.73) $\displaystyle\hskip
40.00006pt-\mathbb{E}_{\mu}\left[\psi\left(\\{(\bm{T}^{G,L}_{t,h})^{1/2}W^{\mathtt{g,L}}_{t,h}\\}_{h=1}^{m},\\{\widetilde{Y}_{0,\ell}\\}_{\ell=1}^{\widetilde{m}},\\{Y_{0,h}\\}_{h=1}^{m},\\{U_{h}\\}_{h=1}^{m},\\{\widetilde{U}_{\ell}\\}_{\ell=1}^{\widetilde{m}}\right)\right]\Bigg{|}=0,$
(C.74)

and

$\displaystyle\lim_{p_{h}\rightarrow\infty}\Bigg{|}\frac{1}{|\mathcal{F}_{R,h}|}\sum_{i\in\mathcal{F}_{R,h}}\phi_{h}\left(({\bm{N}}^{G,R}_{t,h})_{i*},(\bm{V}_{0,h})_{i*},(\bm{V}_{h})_{i*}\right)-\mathbb{E}\left[\phi\left((\bm{T}^{G,R}_{t,h})^{1/2}W^{\mathtt{g,R}}_{t,h},V_{0,h},V_{h}\right)\right]\Bigg{|}=0.$
(C.75)

Here, $W^{\mathtt{g,L}}_{t,1},\ldots,W^{\mathtt{g,L}}_{t,m}$ and
$W^{\mathtt{g,R}}_{t,1},\ldots,W^{\mathtt{g,R}}_{t,\widetilde{m}}$ are
independent; $W^{\mathtt{g,L}}_{t,h},W^{\mathtt{g,R}}_{t,h}\sim
N_{r_{h}}(0,\bm{I}_{r_{h}})$ for $h\in[m]$, $V_{h}\sim\nu_{h}$,
$(U_{1},\ldots,U_{m},\widetilde{U}_{1},\ldots,\widetilde{U}_{\widetilde{m}})\sim\mu$
and for all $h\in[m]$, the parameters $\bm{T}^{G,L}_{t,h}$ and
$\bm{T}^{G,R}_{t,h}$ are defined as follows:

$\displaystyle\bm{T}^{G,L}_{t,h}=\gamma_{h}\mathbb{E}[v^{\mathtt{g}}_{t,h}((\bm{T}^{G,R}_{t,h})^{1/2}W^{\mathtt{g,R}}_{t,h})^{\otimes
2}],\quad\mbox{and}$ (C.76)
$\displaystyle\bm{T}^{G,R}_{t+1,h}=\mathbb{E}[u^{\mathtt{g}}_{t,h}((\bm{T}^{G,L}_{t,1})^{1/2}W^{\mathtt{g,L}}_{t,1},\ldots,(\bm{T}^{G,L}_{t,m})^{1/2}W^{\mathtt{g,L}}_{t,m},\widetilde{Y}_{0,1},\ldots,\widetilde{Y}_{0,\widetilde{m}})^{\otimes
2}],$ (C.77)

where $\bm{T}^{G,R}_{0,h}=\bm{0}$.

Observe that the AMP orbits defined by (C.59) conform to the set-up described
in Lemma C.3. Hence, we can conclude that for any sequence of pseudo-Lipschitz
functions
$\widetilde{\psi}:\mathbb{R}^{3r+2\widetilde{r}}\rightarrow\mathbb{R}$ and
$\widetilde{\phi}_{h}:\mathbb{R}^{3r_{h}}\rightarrow\mathbb{R}$, almost surely
we have:

$\displaystyle\lim_{N\rightarrow\infty}\Bigg{|}\frac{1}{|\mathcal{F}_{L}|}\sum_{i\in\mathcal{F}_{L}}\tilde{\psi}\left(\\{({\bm{N}}^{\perp,L}_{t,h})_{i*}\\}_{h=1}^{m},\\{({\bm{U}}^{\mathrm{pc}}_{0,h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{X}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}},\\{(\bm{U}_{h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{U}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}}\right)$
(C.78) $\displaystyle\hskip
40.00006pt-\mathbb{E}_{\mu}\left[\widetilde{\psi}\left(\\{(\bm{T}^{L,\perp}_{t,h})^{1/2}W^{\perp,L}_{t,h}\\}_{h=1}^{m},\\{Y_{0,h}\\}_{h=1}^{m},\\{\widetilde{Y}_{0,\ell}\\}_{\ell=1}^{\widetilde{m}},\\{U_{h}\\}_{h=1}^{m},\\{\widetilde{U}_{\ell}\\}_{\ell=1}^{\widetilde{m}}\right)\right]\Bigg{|}=0,$
(C.79)

and

$\displaystyle\lim_{p_{h}\rightarrow\infty}\Bigg{|}\frac{1}{|\mathcal{F}_{R,h}|}\sum_{i\in\mathcal{F}_{R,h}}\widetilde{\phi}_{h}\left(({\bm{N}}^{\perp,R}_{t,h})_{i*},({\bm{V}}^{\mathrm{pc}}_{0,h})_{i*},(\bm{V}_{h})_{i*}\right)-\mathbb{E}_{\nu_{h}}\left[\widetilde{\phi}_{h}\left((\bm{T}^{R,\perp}_{t,h})^{1/2}W^{\perp,R}_{t,h},Y^{R}_{0,h},V_{h}\right)\right]\Bigg{|}=0,$
(C.80)

where $\\{\bm{T}^{L,\perp}_{t,h}:h\in[m]\\}$ and
$\\{\bm{T}^{R,\perp}_{t,h}:h\in[m]\\}$ are defined by (C.20). Let us choose
the following sequence of pseudo-Lipschitz functions:

$\displaystyle\tilde{\psi}\left(\\{({\bm{N}}^{\perp,L}_{t,h})_{i*}\\}_{h=1}^{m},\\{({\bm{U}}^{\mathrm{pc}}_{0,h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{X}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}},\\{(\bm{U}_{h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{U}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}}\right)$
(C.81) $\displaystyle\hskip
10.00002pt=\psi\bigg{(}\\{({\bm{N}}^{\perp,L}_{t,h})_{i*}+\bm{A}^{L,\perp}_{t,h}(\bm{U}_{h})_{i*}+\bm{B}^{L,\perp}_{t,h}({\bm{U}}^{\mathrm{pc}}_{0,h})_{i*}\\}_{h=1}^{m},\\{({\bm{U}}^{\mathrm{pc}}_{0,h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{X}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}},$
(C.82) $\displaystyle\hskip
100.00015pt\\{(\bm{U}_{h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{U}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}}\bigg{)}$
(C.83)
$\displaystyle\tilde{\phi}_{h}\left(({\bm{N}}^{\perp,R}_{t,h})_{i*},({\bm{V}}^{\mathrm{pc}}_{0,h})_{i*},(\bm{V}_{h})_{i*}\right)=\phi_{h}\left(({\bm{N}}^{\perp,R}_{t,h})_{i*}+\bm{A}^{R,\perp}_{t,h}(\bm{V}_{h})_{i*}+\bm{B}^{R,\perp}_{t,h}({\bm{V}}^{\mathrm{pc}}_{0,h})_{i*},({\bm{V}}^{\mathrm{pc}}_{0,h})_{i*},(\bm{V}_{h})_{i*}\right)$
(C.84)

Consequently, for all $h\in[m]$, almost surely we have

$\displaystyle\lim_{N\rightarrow\infty}\Bigg{|}\frac{1}{|\mathcal{F}_{L}|}\sum_{i\in\mathcal{F}_{L}}\psi\bigg{(}\\{({\bm{N}}^{\perp,L}_{t,h}+\bm{U}_{h}(\bm{A}^{L,\perp}_{t,h})^{\top}+{\bm{U}}^{\mathrm{pc}}_{0,h}(\bm{B}^{L,\perp}_{t,h})^{\top})_{i*}\\}_{h=1}^{m},\\{({\bm{U}}^{\mathrm{pc}}_{0,h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{X}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}},$
(C.85) $\displaystyle\hskip
140.00021pt\\{(\bm{U}_{h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{U}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}}\bigg{)}$
(C.86)
$\displaystyle-\mathbb{E}\left[\psi\left(\\{\bm{A}^{L,\perp}_{t,h}U_{h}+\bm{B}^{L,\perp}_{t,h}Y_{0,h}+(\bm{T}^{L,\perp}_{t,h})^{1/2}W^{\perp,L}_{t,h}\\}_{h=1}^{m},\\{Y_{0,h}\\}_{h=1}^{m},\\{\widetilde{Y}_{0,\ell}\\}_{\ell=1}^{\widetilde{m}},\\{U_{h}\\}_{h=1}^{m},\\{\widetilde{U}_{\ell}\\}_{\ell=1}^{\widetilde{m}}\right)\right]\Bigg{|}=0,$
(C.87)

and

$\displaystyle\lim_{p_{h}\rightarrow\infty}\Bigg{|}\frac{1}{|\mathcal{F}_{R,h}|}\sum_{i\in\mathcal{F}_{R,h}}\phi_{h}\left(({\bm{N}}^{\perp,R}_{t,h})_{i*},({\bm{V}}^{\mathrm{pc}}_{0,h})_{i*},(\bm{V}_{h})_{i*}\right)$
(C.88) $\displaystyle\hskip
40.00006pt-\mathbb{E}_{\nu_{h}}\left[\phi_{h}\left(\bm{A}^{R,\perp}_{t,h}V_{h}+\bm{B}^{R,\perp}_{t,h}Y^{R}_{0,h}+(\bm{T}^{R,\perp}_{t,h})^{1/2}W^{\perp,R}_{t,h},Y^{R}_{0,h},V_{h}\right)\right]\Bigg{|}=0.$
(C.89)

So, to establish the result, it is enough to show:

$\displaystyle\lim_{N\rightarrow\infty}\Bigg{|}\frac{1}{|\mathcal{F}_{L}|}\sum_{i\in\mathcal{F}_{L}}\psi\bigg{(}\\{({\bm{N}}^{\perp,L}_{t,h}+\bm{U}_{h}(\bm{A}^{L,\perp}_{t,h})^{\top}+{\bm{U}}^{\mathrm{pc}}_{0,h}(\bm{B}^{L,\perp}_{t,h})^{\top})_{i*}\\}_{h=1}^{m},\\{({\bm{U}}^{\mathrm{pc}}_{0,h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{X}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}},$
(C.90) $\displaystyle\hskip
140.00021pt\\{(\bm{U}_{h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{U}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}}\bigg{)}$
(C.91) $\displaystyle\hskip
40.00006pt-\frac{1}{|\mathcal{F}_{L}|}\sum_{i\in\mathcal{F}_{L}}\psi\bigg{(}\\{(\bm{U}^{\perp}_{t,h})_{i*}\\}_{h=1}^{m},\\{({\bm{U}}^{\mathrm{pc}}_{0,h})_{i*}\\}_{d=1}^{m},\\{(\widetilde{\bm{X}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}},\\{(\bm{U}_{h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{U}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}}\bigg{)}\Bigg{|}=0,$
(C.92)

and

$\displaystyle\lim_{p_{h}\rightarrow\infty}\Bigg{|}\frac{1}{|\mathcal{F}_{R,h}|}\sum_{i\in\mathcal{F}_{R,h}}\phi_{h}\left(({\bm{N}}^{\perp,R}_{t,h}+\bm{V}_{h}(\bm{A}^{R,\perp}_{t,h})^{\top}+{\bm{V}}^{\mathrm{pc}}_{0,h}(\bm{B}^{R,\perp}_{t,h})^{\top})_{i*},({\bm{V}}^{\mathrm{pc}}_{0,h})_{i*},(\bm{V}_{h})_{i*}\right)$
(C.93) $\displaystyle\hskip
40.00006pt-\frac{1}{|\mathcal{F}_{R,h}|}\sum_{i\in\mathcal{F}_{R,h}}\phi_{h}\left((\bm{V}^{\perp}_{t,h})_{i*},({\bm{V}}^{\mathrm{pc}}_{0.h})_{i*},(\bm{V}_{h})_{i*}\right)\Bigg{|}=0.$
(C.94)

To show (C.90) and (C.93), for $h\in[m]$, we define the discrepancy between
the sets of iterates $\\{\bm{U}^{\perp}_{t,h},\bm{V}^{\perp}_{t,h}\\}_{t\geq
0}$ and
$\\{{\bm{N}}^{\perp,L}_{t,h}+\bm{U}_{h}(\bm{A}^{L,\perp}_{t,h})^{\top}+{\bm{U}}^{\mathrm{pc}}_{0,h}(\bm{B}^{L,\perp}_{t,h})^{\top},{\bm{N}}^{\perp,R}_{t,h}+\bm{V}_{h}(\bm{A}^{R,\perp}_{t,h})^{\top}+{\bm{V}}^{\mathrm{pc}}_{0,h}(\bm{B}^{R,\perp}_{t,h})^{\top}\\}_{t\geq
0}$ as follows.

$\displaystyle\bm{\Delta}_{1,t,h}$
$\displaystyle=\bm{U}^{\perp}_{t,h}-({\bm{N}}^{\perp,L}_{t,h}+\bm{U}_{h}(\bm{A}^{L,\perp}_{t,h})^{\top}+{\bm{U}}^{\mathrm{pc}}_{0,h}(\bm{B}^{L,\perp}_{t,h})^{\top}),$
(C.95) $\displaystyle\bm{\Delta}_{2,t,h}$
$\displaystyle=\bm{V}^{\perp}_{t,h}-({\bm{N}}^{\perp,R}_{t,h}+\bm{V}_{h}(\bm{A}^{R,\perp}_{t,h})^{\top}+{\bm{V}}^{\mathrm{pc}}_{0,h}(\bm{B}^{R,\perp}_{t,h})^{\top}).$
(C.96)

We shall inductively show the following:

  1. 1.

For any sequence of deterministic subsets $\mathcal{F}_{L}\subset[N]$
satisfying $\frac{|\mathcal{F}_{L}|}{N}\rightarrow\lambda_{L}\in(0,1]$, as
$N\rightarrow\infty$ and pseudo-Lipschitz function
$\psi:\mathbb{R}^{3r+2\widetilde{r}}\rightarrow\mathbb{R}$, for all $t\geq-1$,
almost surely

$\displaystyle\lim_{N\rightarrow\infty}\Bigg{|}\frac{1}{|\mathcal{F}_{L}|}\sum_{i\in\mathcal{F}_{L}}\psi\bigg{(}\\{({\bm{N}}^{\perp,L}_{t,h}+\bm{U}_{h}(\bm{A}^{L,\perp}_{t,h})^{\top}+{\bm{U}}^{\mathrm{pc}}_{0,h}(\bm{B}^{L,\perp}_{t,h})^{\top})_{i*}\\}_{h=1}^{m},\\{({\bm{U}}^{\mathrm{pc}}_{0,h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{X}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}},$
(C.97) $\displaystyle\hskip
140.00021pt\\{(\bm{U}_{h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{U}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}}\bigg{)}$
(C.98) $\displaystyle\hskip
10.00002pt-\frac{1}{|\mathcal{F}_{L}|}\sum_{i\in\mathcal{F}_{L}}\psi\left(\\{(\bm{U}^{\perp}_{t,h})_{i*}\\}_{h=1}^{m},\\{({\bm{U}}^{\mathrm{pc}}_{0,h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{X}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}},\\{(\bm{U}_{h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{U}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}}\right)\Bigg{|}=0,$
(C.99)

  2. 2.

For any sequence of deterministic subsets $\mathcal{F}_{R,h}\subset[p_{h}]$
satisfying
$\frac{|\mathcal{F}_{R,h}|}{p_{h}}\rightarrow\lambda_{R,h}\in(0,1]$, as
$p_{h}\rightarrow\infty$ and any pseudo-Lipschitz functions
$\phi_{h}:\mathbb{R}^{3r_{h}}\rightarrow\mathbb{R}$ for $h\in[m]$, for all
$t\geq-1$, almost surely

$\displaystyle\lim_{p_{h}\rightarrow\infty}\Bigg{|}\frac{1}{|\mathcal{F}_{R,h}|}\sum_{i\in\mathcal{F}_{R,h}}\phi_{h}\left(({\bm{N}}^{\perp,R}_{t,h}+\bm{V}_{h}(\bm{A}^{R,\perp}_{t,h})^{\top}+{\bm{V}}^{\mathrm{pc}}_{0,h}(\bm{B}^{R,\perp}_{t,h})^{\top})_{i*},({\bm{V}}^{\mathrm{pc}}_{0,h})_{i*},(\bm{V}_{h})_{i*}\right)$
(C.100) $\displaystyle\hskip
40.00006pt-\frac{1}{|\mathcal{F}_{R,h}|}\sum_{i\in\mathcal{F}_{R,h}}\phi_{h}\left((\bm{V}^{\perp}_{t,h})_{i*},(\bm{V}_{0.h})_{i*},(\bm{V}_{h})_{i*}\right)\Bigg{|}=0.$
(C.101)

  3. 3.

For $t\geq-1$, we have

$\lim\limits_{N\rightarrow\infty}\frac{1}{N}\|\bm{\Delta}_{1,t,h}\|^{2}_{F}=0.$

  4. 4.

For $t\geq-1$ and for all $h\in[m]$, we have

$\lim\limits_{p_{h}\rightarrow\infty}\frac{1}{p_{h}}\|\bm{\Delta}_{2,t,h}\|^{2}_{F}=0.$

  5. 5.

For $t\geq-1$, we have

$\lim\limits_{N\rightarrow\infty}\frac{1}{N}\|\bm{U}^{\perp}_{t,h}\|^{2}_{F}<\infty,\quad\mbox{and}\quad\lim\limits_{N\rightarrow\infty}\frac{1}{N}\|{\bm{N}}^{\perp,L}_{t,h}+\bm{U}_{h}(\bm{A}^{L,\perp}_{t,h})^{\top}+{\bm{U}}^{\mathrm{pc}}_{0,h}(\bm{B}^{L,\perp}_{t,h})^{\top}\|^{2}_{F}<\infty.$

  6. 6.

For $t\geq 0$ and $h\in[m]$, we have

$\lim\limits_{p_{h}\rightarrow\infty}\frac{1}{p_{h}}\|\bm{V}^{\perp}_{t+1,h}\|^{2}_{F}<\infty,\quad\mbox{and}\quad\lim\limits_{p_{h}\rightarrow\infty}\frac{1}{p_{h}}\|{\bm{N}}^{\perp,R}_{t+1,h}+\bm{V}_{h}(\bm{A}^{R,\perp}_{t+1,h})^{\top}+{\bm{V}}^{\mathrm{pc}}_{0,h}(\bm{B}^{R,\perp}_{t+1,h})^{\top}\|^{2}_{F}<\infty.$

To show the above assertions, we first introduce the following notations:

$\displaystyle
f_{t,h}(\bm{U}^{\perp}_{t,1},\ldots,\bm{U}^{\perp}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}})=u^{\circ}_{t,h}(\bm{U}^{\perp}_{t,1},\ldots,\bm{U}^{\perp}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}})-\bm{P}_{{\bm{U}}^{\mathrm{pc}}_{0,h}}\widebar{\bm{U}}^{\perp}_{t,h},$
(C.102) $\displaystyle
g_{t,h}(\bm{V}^{\perp}_{t,h})=v^{\circ}_{t,h}(\bm{V}^{\perp}_{t,h})-\bm{P}_{{\bm{V}}^{\mathrm{pc}}_{0,h}}\widebar{\bm{V}}^{\perp}_{t,h},$
(C.103)
$\displaystyle\bm{\delta}_{t,1,h}=\bm{P}_{{\bm{V}}^{\mathrm{pc}}_{0,h}}(\bm{W}^{\perp}_{h})^{\top}f_{t,h}(\bm{U}^{\perp}_{t,1},\ldots,\bm{U}^{\perp}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}}),\quad\bm{\delta}_{t,2,h}=\bm{P}_{{\bm{U}}^{\mathrm{pc}}_{0,h}}\bm{W}^{\perp}_{h}g_{t,h}(\bm{V}^{\perp}_{t,h}).$
(C.104)

Here,
$\bm{P}_{{\bm{U}}^{\mathrm{pc}}_{0,h}}=N^{-1}{\bm{U}}^{\mathrm{pc}}_{0,h}({\bm{U}}^{\mathrm{pc}}_{0,h})^{\top}$
and
$\bm{P}_{{\bm{V}}^{\mathrm{pc}}_{0,h}}=p_{h}^{-1}{\bm{V}}^{\mathrm{pc}}_{0,h}({\bm{V}}^{\mathrm{pc}}_{0,h})^{\top}$.
Observe that for $t\geq 0$ and $h\in[m]$:

$\displaystyle\bm{U}^{\perp}_{t,h}$
$\displaystyle={\bm{U}}^{\mathrm{pc}}_{0,h}\bigg{[}\bm{D}_{0,h}\frac{({\bm{V}}^{\mathrm{pc}}_{0,h})^{\top}v^{\circ}_{t,h}(\bm{V}^{\perp}_{t,h})}{N}-\frac{({\bm{U}}^{\mathrm{pc}}_{0,h})^{\top}\widebar{\bm{U}}^{\perp}_{t-1,h}}{N}\gamma_{h}\left(\bm{J}^{R,\circ}_{t,h}(\bm{V}^{\perp}_{t,h};\nu_{h})\right)^{\top}$
(C.105) $\displaystyle\hskip
100.00015pt-\frac{({\bm{U}}^{\mathrm{pc}}_{0,h})^{\top}\bm{U}_{h}}{N}\bigg{(}\frac{\bm{D}_{h}\bm{V}^{\top}_{h}\bm{P}^{\perp}_{{\bm{V}}^{\mathrm{pc}}_{0,h}}v^{\circ}_{t,h}(\bm{V}^{\perp}_{t,h})}{N}\bigg{)}\bigg{]}$
(C.106) $\displaystyle\hskip
10.00002pt+\frac{1}{N}\bm{U}_{h}\bm{D}_{h}\bm{V}^{\top}_{h}\bm{P}^{\perp}_{{\bm{V}}^{\mathrm{pc}}_{0,h}}v^{\circ}_{t,h}(\bm{V}^{\perp}_{t,h})-f_{t-1,h}(\bm{U}^{\perp}_{t-1,1},\ldots,\bm{U}^{\perp}_{t-1,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}})\gamma_{h}\left(\bm{J}^{R,\circ}_{t,h}(\bm{V}^{\perp}_{t,h};\nu_{h})\right)^{\top}$
(C.107) $\displaystyle\hskip
10.00002pt+\bm{W}^{\perp}_{h}g_{t,h}(\bm{V}^{\perp}_{t,h})-\bm{\delta}_{t,2,h},$
(C.108) $\displaystyle\bm{V}^{\perp}_{t,h}$
$\displaystyle={\bm{V}}^{\mathrm{pc}}_{0,h}\bigg{[}\bm{D}_{0,h}\frac{({\bm{U}}^{\mathrm{pc}}_{0,h})^{\top}u^{\circ}_{t,h}(\bm{U}^{\perp}_{t,1},\ldots,\bm{U}^{\perp}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}})}{N}$
(C.109) $\displaystyle\hskip
50.00008pt-\frac{({\bm{V}}^{\mathrm{pc}}_{0,h})^{\top}\widebar{\bm{V}}^{\perp}_{t,h}}{p_{h}}\left(\bm{J}^{L,\circ}_{t,h}(\bm{U}^{\perp}_{t,1},\ldots,\bm{U}^{\perp}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}})\right)^{\top}$
(C.110) $\displaystyle\hskip
60.00009pt-\frac{({\bm{V}}^{\mathrm{pc}}_{0,h})^{\top}\bm{V}_{h}}{p_{h}}\bigg{(}\frac{\bm{D}_{h}\bm{U}^{\top}_{h}\bm{P}^{\perp}_{{\bm{U}}^{\mathrm{pc}}_{0,h}}u^{\circ}_{t,h}(\bm{U}^{\perp}_{t,1},\ldots,\bm{U}^{\perp}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}})}{N}\bigg{)}\bigg{]}$
(C.111) $\displaystyle\hskip
10.00002pt+\frac{1}{N}\bm{V}_{h}\bm{D}_{h}\bm{U}^{\top}_{h}\bm{P}^{\perp}_{{\bm{U}}^{\mathrm{pc}}_{0,h}}u^{\circ}_{t,h}(\bm{U}^{\perp}_{t,1},\ldots,\bm{U}^{\perp}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}})+(\bm{W}^{\perp}_{h})^{\top}f_{t,h}(\bm{U}^{\perp}_{t,1},\ldots,\bm{U}^{\perp}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}})$
(C.112) $\displaystyle\hskip 10.00002pt-
g_{t,h}(\bm{V}^{\perp}_{t,h})\left(\bm{J}^{L,\circ}_{t,h}(\bm{U}^{\perp}_{t,1},\ldots,\bm{U}^{\perp}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}})\right)^{\top}-\bm{\delta}_{t,1,h},$
(C.113)

with the convention
$u^{\perp,\circ}_{-1,h}(\bm{U}^{\perp}_{-1,1},\ldots,\bm{U}^{\perp}_{-1,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}})=\widebar{\bm{U}}^{\perp}_{-1,h}$.
Consequently, we have

$\displaystyle\bm{\Delta}_{t,1,h}$
$\displaystyle={\bm{U}}^{\mathrm{pc}}_{0,h}\bigg{[}\bm{D}_{0,h}\frac{({\bm{V}}^{\mathrm{pc}}_{0,h})^{\top}v^{\circ}_{t,h}(\bm{V}^{\perp}_{t,h})}{N}-\frac{({\bm{U}}^{\mathrm{pc}}_{0,h})^{\top}\widebar{\bm{U}}^{\perp}_{t-1,h}}{N}\gamma_{h}\left(\bm{J}^{R,\circ}_{t,h}(\bm{V}^{\perp}_{t,h};\nu_{h})\right)^{\top}$
(C.114) $\displaystyle\hskip
10.00002pt-\frac{({\bm{U}}^{\mathrm{pc}}_{0,h})^{\top}\bm{U}_{h}}{N}\bigg{(}\frac{\bm{D}_{h}\bm{V}^{\top}_{h}\bm{P}^{\perp}_{{\bm{V}}^{\mathrm{pc}}_{0,h}}v^{\circ}_{t,h}(\bm{V}^{\perp}_{t,h})}{N}\bigg{)}-(\bm{B}^{L,\perp}_{t,h})^{\top}\bigg{]}+\bm{U}_{h}\bigg{[}\bm{D}_{h}\frac{\bm{V}^{\top}_{h}\bm{P}^{\perp}_{{\bm{V}}^{\mathrm{pc}}_{0,h}}v^{\circ}_{t,h}(\bm{V}^{\perp}_{t,h})}{N}-(\bm{A}^{L,\perp}_{t,h})^{\top}\bigg{]}$
(C.115) $\displaystyle\hskip
20.00003pt+\bm{W}^{\perp}_{h}(g_{t,h}(\bm{V}^{\perp}_{t,h})-v^{\perp,\circ}_{t,h}({\bm{N}}^{\perp,R}_{t,h}+\bm{V}_{h}(\bm{A}^{R,\perp}_{t,h})^{\top}+{\bm{V}}^{\mathrm{pc}}_{0,h}(\bm{B}^{R,\perp}_{t,h})^{\top}))$
(C.116) $\displaystyle\hskip
20.00003pt+\bigg{(}\widebar{\bm{N}}^{\perp,L}_{t-1,h}\,\gamma_{h}\left(\bm{J}^{R,\perp}_{t,h}({\bm{N}}^{\perp,R}_{t,h}+\bm{V}_{h}(\bm{A}^{R,\perp}_{t,h})^{\top}+{\bm{V}}^{\mathrm{pc}}_{0,h}(\bm{B}^{R,\perp}_{t,h})^{\top};\nu_{h})\right)^{\top}$
(C.117) $\displaystyle\hskip 40.00006pt-
f_{t-1,h}(\bm{U}^{\perp}_{t-1,1},\ldots,\bm{U}^{\perp}_{t-1,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}})\gamma_{h}\left(\bm{J}^{R,\circ}_{t,h}(\bm{V}^{\perp}_{t,h};\nu_{h})\right)^{\top}\bigg{)}-\bm{\delta}_{t,2,h}$
(C.118) $\displaystyle\bm{\Delta}_{t+1,2,h}$
$\displaystyle={\bm{V}}^{\mathrm{pc}}_{0,h}\bigg{[}\bm{D}_{0,h}\frac{({\bm{U}}^{\mathrm{pc}}_{0,h})^{\top}u^{\circ}_{t,h}(\bm{U}^{\perp}_{t,1},\ldots,\bm{U}^{\perp}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}})}{N}$
(C.119) $\displaystyle\hskip
20.00003pt-\frac{({\bm{V}}^{\mathrm{pc}}_{0,h})^{\top}\widebar{\bm{V}}^{\perp}_{t,h}}{p_{h}}(\bm{J}^{L,\circ}_{t,h}(\bm{U}^{\perp}_{t,1},\ldots,\bm{U}^{\perp}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}},\mu))^{\top}$
(C.120) $\displaystyle\hskip
20.00003pt-\frac{({\bm{V}}^{\mathrm{pc}}_{0,h})^{\top}\bm{V}_{h}}{p_{h}}\bigg{(}\frac{\bm{D}_{h}\bm{U}^{\top}_{h}\bm{P}^{\perp}_{{\bm{U}}^{\mathrm{pc}}_{0,h}}u^{\circ}_{t,h}(\bm{U}^{\perp}_{t,1},\ldots,\bm{U}^{\perp}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}})}{N}\bigg{)}-(\bm{B}^{R,\perp}_{t,h})^{\top}\bigg{]}$
(C.121)
$\displaystyle+\bm{V}_{h}\bigg{[}\bm{D}_{h}\frac{\bm{U}^{\top}_{h}\bm{P}^{\perp}_{{\bm{U}}^{\mathrm{pc}}_{0,h}}u^{\circ}_{t,h}(\bm{U}^{\perp}_{t,1},\ldots,\bm{U}^{\perp}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}})}{N}-(\bm{A}^{R,\perp}_{t,h})^{\top}\bigg{]}-\bm{\delta}_{t,2,h}$
(C.122)
$\displaystyle+(\bm{W}^{\perp}_{h})^{\top}\bigg{(}f_{t,h}(\bm{U}^{\perp}_{t,1},\ldots,\bm{U}^{\perp}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}})$
(C.123)
$\displaystyle-u^{\perp,\circ}_{t,h}({\bm{N}}^{\perp,L}_{t,1}+\bm{U}_{h}(\bm{A}^{L,\perp}_{t,1})^{\top}+{\bm{U}}^{\mathrm{pc}}_{0,1}(\bm{B}^{L,\perp}_{t,1})^{\top},\ldots,{\bm{N}}^{\perp,L}_{t,m}+\bm{U}_{m}(\bm{A}^{L,\perp}_{t,m})^{\top}+{\bm{U}}^{\mathrm{pc}}_{0,m}(\bm{B}^{L,\perp}_{t,m})^{\top},$
(C.124) $\displaystyle\hskip
300.00046pt\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}})\bigg{)}$
(C.125)
$\displaystyle+\bigg{(}\widebar{\bm{N}}^{\perp,R}_{t,h}\,\bigg{(}\bm{J}^{L,\perp}_{t,h}({\bm{N}}^{\perp,L}_{t,1}+\bm{U}_{h}(\bm{A}^{L,\perp}_{t,h})^{\top}+{\bm{U}}^{\mathrm{pc}}_{0,h}(\bm{B}^{L,\perp}_{t,h})^{\top},\ldots,$
(C.126) $\displaystyle\hskip
100.00015pt{\bm{N}}^{\perp,L}_{t,h}+\bm{U}_{h}(\bm{A}^{L,\perp}_{t,h})^{\top}+{\bm{U}}^{\mathrm{pc}}_{0,h}(\bm{B}^{L,\perp}_{t,h})^{\top},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}})\bigg{)}^{\top}$
(C.127) $\displaystyle\hskip 100.00015pt-
g_{t,h}(\bm{V}^{\perp}_{t,h})\left(\bm{J}^{L,\circ}_{t,h}(\bm{U}^{\perp}_{t,1},\ldots,\bm{U}^{\perp}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}})\right)^{\top}\bigg{)}$
(C.128)

The base case for $t=-1$ follows by the definitions of $\bm{U}^{\perp}_{-1,h}$
and $\bm{V}^{\perp}_{0,h}$. Let us assume that the assertions (1)-(6) hold for
$t=-1,\ldots,r$. First, let us consider Assertion (3) for $t=r+1$. Using
(C.20), the assertions (2), (4), (6) for $t=r$ and arguments similar to the
proof of (B.46) of Montanari and Venkataramanan, [31], we can show

$\displaystyle\lim_{N\rightarrow\infty}\bigg{[}\bm{D}_{0,h}\frac{({\bm{V}}^{\mathrm{pc}}_{0,h})^{\top}v^{\circ}_{t,h}(\bm{V}^{\perp}_{t,h})}{N}-\frac{({\bm{U}}^{\mathrm{pc}}_{0,h})^{\top}\widebar{\bm{U}}^{\perp}_{t-1,h}}{N}\gamma_{h}\left(\bm{J}^{R,\circ}_{t,h}(\bm{V}^{\perp}_{t,h};\nu_{h})\right)^{\top}$
(C.130) $\displaystyle\hskip
40.00006pt-\frac{({\bm{U}}^{\mathrm{pc}}_{0,h})^{\top}\bm{U}_{h}}{N}\bigg{(}\frac{\bm{D}_{h}\bm{V}^{\top}_{h}\bm{P}^{\perp}_{{\bm{V}}^{\mathrm{pc}}_{0,h}}v^{\circ}_{t,h}(\bm{V}^{\perp}_{t,h})}{N}\bigg{)}-(\bm{B}^{L,\perp}_{t,h})^{\top}\bigg{]}=0$
(C.131)

Using the arguments similar to the proof of (B.47) of Montanari and
Venkataramanan, [31] and (C.20), we can further show that for all $h\in[m]$

$\displaystyle\lim_{N\rightarrow\infty}\bigg{[}\bm{D}_{h}\frac{\bm{V}^{\top}_{h}\bm{P}^{\perp}_{{\bm{V}}^{\mathrm{pc}}_{0,h}}v^{\circ}_{t,h}(\bm{V}^{\perp}_{t,h})}{N}-(\bm{A}^{L,\perp}_{t,h})^{\top}\bigg{]}=0.$
(C.132)

Using the fact that $\|\bm{W}^{\perp}_{h}\|^{2}\rightarrow 4$ along with the
Lipschitz property of the functions $v^{\circ}_{0,h}$’s and their derivatives
and the assertions (2), (4), (5), (6) for $t=r-1$, we can show along the lines
of the proof of (B.48) and (B.49) of Montanari and Venkataramanan, [31] that

$\lim\limits_{N\rightarrow\infty}\frac{1}{N}\|\bm{W}^{\perp}_{h}(g_{t,h}(\bm{V}^{\perp}_{t,h})-v^{\perp,\circ}_{t,h}({\bm{N}}^{\perp,R}_{t,h}+\bm{V}_{h}(\bm{A}^{R,\perp}_{t,h})^{\top}+{\bm{V}}^{\mathrm{pc}}_{0,h}(\bm{B}^{R,\perp}_{t,h})^{\top}))\|_{F}=0,$

and

$\displaystyle\lim\limits_{N\rightarrow\infty}\frac{1}{N}\bigg{\|}\widebar{\bm{N}}^{\perp,L}_{t-1,h}\,\gamma_{h}\left(\bm{J}^{R,\perp}_{t,h}({\bm{N}}^{\perp,R}_{t,h}+\bm{V}_{h}(\bm{A}^{R,\perp}_{t,h})^{\top}+{\bm{V}}^{\mathrm{pc}}_{0,h}(\bm{B}^{R,\perp}_{t,h})^{\top};\nu_{h})\right)^{\top}$
(C.133) $\displaystyle\hskip 40.00006pt-
f_{t-1,h}(\bm{U}^{\perp}_{t-1,1},\ldots,\bm{U}^{\perp}_{t-1,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}})\gamma_{h}\left(\bm{J}^{R,\circ}_{t,h}(\bm{V}^{\perp}_{t,h};\nu_{h})\right)^{\top}\bigg{\|}_{F}=0.$
(C.134)

Again, using the Lipschitz property of the denoisers and the previously
established assertions, we can prove along the lines of (B.50) of Montanari
and Venkataramanan, [31] that
$\lim\limits_{N\rightarrow\infty}\frac{1}{N}\|\bm{\delta}_{t,2,h}\|=0$. This
proves assertion (3).

Next, we consider Assertion (5) for $t=r+1$. Consider the pseudo Lipschitz
function
$\psi(x,u,u_{0})=\|x+\bm{A}^{L,\perp}_{t,h}u+\bm{B}^{L,\perp}_{t,h}u_{0}\|^{2}$.
Then by Lemma C.3, we have

$\lim\limits_{N\rightarrow\infty}\frac{1}{N}\|{\bm{N}}^{\perp,L}_{t,h}+\bm{U}_{h}(\bm{A}^{L,\perp}_{t,h})^{\top}+{\bm{U}}^{\mathrm{pc}}_{0,h}(\bm{B}^{L,\perp}_{t,h})^{\top}\|^{2}_{F}<\infty.$

Then, using Assertion (3) for $t=r+1$, we get

$\lim\limits_{N\rightarrow\infty}\frac{1}{N}\|\bm{U}^{\perp}_{t,h}\|^{2}_{F}<\infty.$

To prove assertion (1), observe that, since
$\frac{|\mathcal{F}_{L}|}{N}\rightarrow\lambda_{L}\in(0,1]$, using Cauchy-
Schwarz inequality and properties of pseudo Lipschitz functions (4.6), we can
get a constant $C>0$ (possibly depending on $\lambda_{L}$),

$\displaystyle\lim_{N\rightarrow\infty}\Bigg{|}\frac{1}{|\mathcal{F}_{L}|}\sum_{i\in\mathcal{F}_{L}}\psi\bigg{(}\\{({\bm{N}}^{\perp,L}_{t,h}+\bm{U}_{h}(\bm{A}^{L,\perp}_{t,h})^{\top}+{\bm{U}}^{\mathrm{pc}}_{0,h}(\bm{B}^{L,\perp}_{t,h})^{\top})_{i*}\\}_{h=1}^{m},\\{({\bm{U}}^{\mathrm{pc}}_{0,h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{X}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}},$
(C.135) $\displaystyle\hskip
140.00021pt\\{(\bm{U}_{h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{U}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}}\bigg{)}$
(C.136) $\displaystyle\hskip
40.00006pt-\frac{1}{|\mathcal{F}_{L}|}\sum_{i\in\mathcal{F}_{L}}\psi\left(\\{(\widebar{\bm{U}}^{\perp}_{t,h})_{i*}\\}_{h=1}^{m},\\{({\bm{U}}^{\mathrm{pc}}_{0,h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{X}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}},\\{(\bm{U}_{h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{U}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}}\right)\Bigg{|}$
(C.137) $\displaystyle\leq
C\bigg{(}1+\sum_{h=1}^{m}\frac{\|\bm{\Delta}_{t,h}\|_{F}}{\sqrt{N}}+\sum_{h=1}^{m}\frac{\|\bm{U}_{h}\|_{F}}{\sqrt{N}}+\sum_{h=1}^{m}\frac{\|{\bm{U}}^{\mathrm{pc}}_{0,h}\|_{F}}{\sqrt{N}}+\sum_{\ell=1}^{\widetilde{m}}\frac{\|\widetilde{\bm{X}}_{\ell}\|_{F}}{\sqrt{N}}+\sum_{\ell=1}^{\widetilde{m}}\frac{\|\widetilde{\bm{U}}_{\ell}\|_{F}}{\sqrt{N}}$
(C.138) $\displaystyle\hskip
50.00008pt+\sum_{h=1}^{m}\frac{\|{\bm{N}}^{\perp,L}_{t,h}+\bm{U}_{h}(\bm{A}^{L,\perp}_{t,h})^{\top}+{\bm{U}}^{\mathrm{pc}}_{0,h}(\bm{B}^{L,\perp}_{t,h})^{\top}\|_{F}}{\sqrt{N}}\bigg{)}\times\sum_{h=1}^{m}\frac{\|\bm{\Delta}_{t,h}\|_{F}}{\sqrt{N}}.$
(C.139)

Combining Assertions (3) and (5) for $t=r+1$ with Proposition 4.1, Assertion
(1) follows. The proof of Assertion (4) follows by retracing the steps of the
proof of Assertion (3) while that of Assertion (6) follows using Assertion (3)
and the techniques of the proof of Assertion (5). Finally, we can show
Assertion (2) using Assertions (4) and (6) and the steps of the proof of
Assertion (1).

###  C.4 Proof of Lemma C.3

To prove this lemma, we shall use the Gaussian conditioning technique from
Bolthausen, [7] as in Berthier et al., [6]. Let us denote the $\sigma$-algebra
generated by

$\\{\widebar{\bm{N}}^{G,L}_{0,h},\ldots,\widebar{\bm{N}}^{G,L}_{t-1,h},{\bm{N}}^{G,L}_{0,h},\ldots,{\bm{N}}^{G,L}_{t-1,h},\widebar{\bm{N}}^{G,R}_{1,h},\ldots,\widebar{\bm{N}}^{G,R}_{t,h},{\bm{N}}^{G,R}_{1,h},\ldots,{\bm{N}}^{G,R}_{t,h},\widetilde{\bm{X}}_{\ell}:h\in[m],\,\ell\in[\widetilde{m}]\\}$

by $\mathcal{G}_{t,t}$ and the $\sigma$-algebra generated by

$\\{\widebar{\bm{N}}^{G,R}_{1,h},\ldots,\widebar{\bm{N}}^{G,R}_{t,h},{\bm{N}}^{G,R}_{1,h},\ldots,{\bm{N}}^{G,R}_{t,h},\widebar{\bm{N}}^{G,L}_{0,h},\ldots,\widebar{\bm{N}}^{G,L}_{t,h},{\bm{N}}^{G,L}_{0,h},\ldots,{\bm{N}}^{G,L}_{t,h},\widetilde{\bm{X}}_{\ell}:h\in[m],\,\ell\in[\widetilde{m}]\\}$

by $\mathcal{G}_{t+1,t}$. Let us simplify the notations by introducing the
following notations:

$\bm{J}^{G,R}_{t,h}=\gamma_{h}J^{\mathtt{g,R}}_{t,h}({\bm{N}}^{G,R}_{t,h}),\quad\bm{J}^{G,L}_{t,h}=J^{\mathtt{g,L}}_{t,h}({\bm{N}}^{G,L}_{t,1},\ldots,{\bm{N}}^{G,L}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}}).$

Observe that conditional distribution of $\bm{G}_{h}$ given
$\mathcal{G}_{t+1,t}$ is equivalent to the conditional distribution of
$\bm{G}_{h}$ given

$\underbracket{[{\bm{N}}^{G,R}_{1,h}+\widebar{\bm{N}}^{G,R}_{0,h}(\bm{J}^{G,L}_{0,h})^{\top}|\cdots|{\bm{N}}^{G,R}_{t,h}+\widebar{\bm{N}}^{G,R}_{t-1,h}(\bm{J}^{G,L}_{t-1,h})^{\top}]}_{=\boldsymbol{\mathsf{N}}^{G,R}_{t,h}}=\bm{G}^{\top}_{h}\underbracket{[\widebar{\bm{N}}^{G,L}_{0,h}|\cdots|\widebar{\bm{N}}^{G,L}_{t-1,h}]}_{=\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h}}$

and

$\underbracket{[{\bm{N}}^{G,L}_{0,h}|\cdots|{\bm{N}}^{G,L}_{t,h}+\widebar{\bm{N}}^{G,L}_{t-1,h}(\bm{J}^{G,R}_{t,h})^{\top}]}_{=\boldsymbol{\mathsf{N}}^{G,L}_{t+1,h}}=\bm{G}_{h}\underbracket{[\widebar{\bm{N}}^{G,R}_{0,h}|\cdots|\widebar{\bm{N}}^{G,R}_{t,h}]}_{=\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t+1,h}}.$

Using Lemma 11 and 12 of [4], we can show that

$\bm{G}_{h}|_{\mathcal{G}_{t+1,t}}\overset{d}{=}\boldsymbol{\mathsf{E}}_{t+1,t,h}+\boldsymbol{\mathsf{P}}_{t+1,t,h}(\widetilde{\bm{G}}_{h}),$

where

$\displaystyle\boldsymbol{\mathsf{E}}_{t+1,t,h}=\boldsymbol{\mathsf{N}}^{G,L}_{t+1,h}((\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t+1,h})^{\top}\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t+1,h})^{-1}(\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t+1,h})^{\top}+\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h}((\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h})^{\top}\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h})^{-1}\boldsymbol{\mathsf{N}}^{G,R}_{t,h}$
(C.140) $\displaystyle\hskip
80.00012pt-\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h}((\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h})^{\top}\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h})^{-1}(\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h})^{\top}\boldsymbol{\mathsf{N}}^{G,L}_{t+1,h}((\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t+1,h})^{\top}\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t+1,h})^{-1}(\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t+1,h})^{\top},$
(C.141)
$\displaystyle\boldsymbol{\mathsf{P}}_{t+1,t,h}(\widetilde{\bm{G}}_{h})=\bm{P}^{\perp}_{\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h}}\widetilde{\bm{G}}_{h}\bm{P}^{\perp}_{\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t+1,h}}.$
(C.142)

Here $\widetilde{\bm{G}}_{h}\overset{d}{=}\bm{G}_{h}$ and
$\widetilde{\bm{G}}_{h}$ is independent of $\bm{G}_{h}$ for all $h\in[m]$.
Furthermore, conditioned on $\mathcal{G}_{t,t}$ the distribution of
$\bm{G}_{h}$ for $h\in[m]$ is given by

$\bm{G}_{h}|_{\mathcal{G}_{t,t}}=\boldsymbol{\mathsf{E}}_{t,t,h}+\boldsymbol{\mathsf{P}}_{t,t,h}(\widetilde{\bm{G}}_{h}),$

where

$\displaystyle\boldsymbol{\mathsf{E}}_{t,t,h}=\boldsymbol{\mathsf{N}}^{G,L}_{t,d}((\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t,h})^{\top}\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t,h})^{-1}(\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t,h})^{\top}+\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h}((\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h})^{\top}\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h})^{-1}\boldsymbol{\mathsf{N}}^{G,R}_{t,h}$
(C.143) $\displaystyle\hskip
80.00012pt-\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h}((\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h})^{\top}\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h})^{-1}(\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h})^{\top}\boldsymbol{\mathsf{N}}^{G,L}_{t,d}((\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t,h})^{\top}\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t,h})^{-1}(\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t,h})^{\top},$
(C.144)
$\displaystyle\boldsymbol{\mathsf{P}}_{t,t,d}(\widetilde{\bm{G}}_{h})=\bm{P}^{\perp}_{\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h}}\widetilde{\bm{G}}_{h}\bm{P}^{\perp}_{\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t,h}}.$
(C.145)

Again, the matrices $\widetilde{\bm{G}}_{h}\overset{d}{=}\bm{G}_{h}$ and
$\widetilde{\bm{G}}_{h}$ are independent of $\bm{G}_{h}$ for all $h\in[m]$.
Note that in this analysis we are only considering the non-degenerate case
when
$(\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h})^{\top}\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h}$
and
$(\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t,h})^{\top}\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t,h}$
are of full rank. The degenerate case can be handled by adding a small random
perturbation to the denoisers as shown in Section 5.4 of Berthier et al., [6].

Define the following matrices:

$\displaystyle\bm{\Xi}^{L}_{t,h}=\bigg{(}\frac{(\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h})^{\top}\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h}}{N}\bigg{)}^{-1}\frac{(\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h})^{\top}\widebar{\bm{N}}^{G,L}_{t,h}}{N},\quad\mbox{and}\quad\bm{\Xi}^{R}_{t,h}=\bigg{(}\frac{(\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t,h})^{\top}\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t,h}}{p_{h}}\bigg{)}^{-1}\frac{(\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t,h})^{\top}\widebar{\bm{N}}^{G,R}_{t,h}}{p_{h}}.$
(C.146)

Following the techniques used in the proof of Lemma E.1 of Ma and Nandy, [26],
we can inductively show the following statements for all $t\geq 0$.

  1. 1.

If
$\boldsymbol{\mathrm{N}}_{L,t,h}=[{\bm{N}}^{G,L}_{0,h}|\cdots|{\bm{N}}^{G,L}_{t-1,h}]$
and
$\boldsymbol{\mathrm{N}}_{R,t,h}=[{\bm{N}}^{G,R}_{1,h}|\cdots|{\bm{N}}^{G,R}_{t,h}]$,
then for $h\in[m]$ and $N\rightarrow\infty$, we have

$\displaystyle{\bm{N}}^{G,L}_{t,h}|_{\mathcal{G}_{t,t}}=\boldsymbol{\mathrm{N}}_{L,t,h}\bm{\Xi}^{R}_{t,h}+\widetilde{\bm{G}}_{h}\bm{P}^{\perp}_{\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t,h}}\widebar{\bm{N}}^{G,R}_{t,h}+\widetilde{\boldsymbol{\mathrm{N}}}_{R,t,h}\,o(1),$
(C.147)
$\displaystyle{\bm{N}}^{G,R}_{t+1,h}|_{\mathcal{G}_{t+1,t}}=\boldsymbol{\mathrm{N}}_{R,t,h}\bm{\Xi}^{L}_{t,h}+\widetilde{\bm{G}}^{\top}_{h}\bm{P}^{\perp}_{\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h}}\widebar{\bm{N}}^{G,L}_{t,h}+\widetilde{\boldsymbol{\mathrm{N}}}_{L,t,h}\,o(1),$
(C.148)

where the matrices $\widetilde{\bm{G}}_{h}\overset{d}{=}\bm{G}_{h}$ and
$\widetilde{\bm{G}}_{h}$ are independent of $\bm{G}_{h}$ for all $h\in[m]$.
Furthermore, for all $h\in[m]$, the matrices
$\widetilde{\boldsymbol{\mathrm{N}}}_{L,t,h}$ and
$\widetilde{\boldsymbol{\mathrm{N}}}_{R,t,h}$ is formed by Gram-Schmidt
orthogonalization of the columns of
$\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h}$ and
$\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t,h}$, respectively.

  2. 2.

For any sequence of deterministic subsets $\mathcal{F}_{L}\subset[N]$
satisfying $\frac{|\mathcal{F}_{L}|}{N}\rightarrow\lambda_{L}\in(0,1]$, as
$N\rightarrow\infty$ and $\mathcal{F}_{R,h}\subset[p_{h}]$, for $h\in[m]$
satisfying
$\frac{|\mathcal{F}_{R,h}|}{p_{h}}\rightarrow\lambda_{R,h}\in(0,1]$, as
$p_{h}\rightarrow\infty$. For any pseudo Lipschitz functions
$\psi:\mathbb{R}^{3r+2\widetilde{r}}\rightarrow\mathbb{R}$ and
$\phi_{h}:\mathbb{R}^{3r_{h}}\rightarrow\mathbb{R}$ for $h\in[m]$, we have:

$\displaystyle\lim_{N\rightarrow\infty}\Bigg{|}\frac{1}{|\mathcal{F}_{L}|}\sum_{i\in\mathcal{F}_{L}}\psi\left(\\{({\bm{N}}^{G,L}_{t,h})_{i*}\\}_{h=1}^{m},\\{({\bm{U}}^{\mathrm{pc}}_{0,h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{X}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}},\\{(\bm{U}_{h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{U}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}}\right)$
$\displaystyle\hskip
40.00006pt-\mathbb{E}\left[\psi\left(\\{(\bm{T}^{G,L}_{t,h})^{1/2}W^{\mathtt{g,L}}_{t,h}\\}_{h=1}^{m},\\{Y_{0,h}\\}_{h=1}^{m},\\{\widetilde{Y}_{\ell}\\}_{\ell=1}^{\widetilde{m}},\\{U_{h}\\}_{h=1}^{m},\\{\widetilde{U}_{\ell}\\}_{\ell=1}^{\widetilde{m}}\right)\right]\Bigg{|}=0,$

and

$\displaystyle\lim_{p_{h}\rightarrow\infty}\Bigg{|}\frac{1}{|\mathcal{F}_{R,h}|}\sum_{i\in\mathcal{F}_{R,h}}\phi_{h}\left(({\bm{N}}^{G,R}_{t,h})_{i*},({\bm{V}}^{\mathrm{pc}}_{0,h})_{i*},(\bm{V}_{h})_{i*}\right)-\mathbb{E}\left[\phi\left((\bm{T}^{G,R}_{t,h})^{1/2}W^{\mathtt{g,R}}_{t,h},Y^{R}_{0,h},V_{h}\right)\right]\Bigg{|}=0.$

  3. 3.

For all $k,\ell\in\\{0,\ldots,t\\}$, we have for $h\in[m]$:

$\displaystyle\lim_{p_{h}\rightarrow\infty}\frac{({\bm{N}}^{G,R}_{k+1,h})^{\top}{\bm{N}}^{G,R}_{\ell+1,h}}{p_{h}}=\lim_{N\rightarrow\infty}\frac{(\widebar{\bm{N}}^{G,L}_{k,h})^{\top}\widebar{\bm{N}}^{G,L}_{\ell,h}}{N},$
(C.151)
$\displaystyle\lim_{N\rightarrow\infty}\frac{({\bm{N}}^{G,L}_{k,h})^{\top}{\bm{N}}^{G,L}_{\ell,h}}{N}=\gamma_{h}\lim_{N\rightarrow\infty}\frac{(\widebar{\bm{N}}^{G,R}_{k,h})^{\top}\widebar{\bm{N}}^{G,R}_{\ell,h}}{N}.$
(C.152)

  4. 4.

For all $h\in[m]$, let
$\varphi_{h}(x_{1},\ldots,x_{m},{\widetilde{x}}_{1},\ldots,{\widetilde{x}}_{\widetilde{m}}):\mathbb{R}^{r+\widetilde{r}}\rightarrow\mathbb{R}$
and $\vartheta_{h}(x):\mathbb{R}^{r_{h}}\rightarrow\mathbb{R}$ be Lipschitz
functions defined on appropriate domains with Lipschitz Jacobians. Let
$\mathsf{d}\varphi_{h}$ and $\mathsf{d}\vartheta_{h}$ be the Jacobians of
$\varphi_{h}$ and $\vartheta_{h}$ with respect to the $x_{h}$ and $y_{h}$
respectively. Then we have the following:

$\displaystyle\lim_{N\rightarrow\infty}\frac{({\bm{N}}^{G,L}_{k,h})^{\top}\varphi_{h}(\\{{\bm{N}}^{G,L}_{\ell,j}\\}_{j=1}^{m},\\{\widetilde{\bm{X}}_{j^{\prime}}\\}_{j^{\prime}=1}^{\widetilde{m}})}{N}$
(C.153) $\displaystyle\hskip
30.00005pt=\lim_{N\rightarrow\infty}\frac{({\bm{N}}^{G,L}_{k,h})^{\top}{\bm{N}}^{G,L}_{\ell,h}}{N}\lim\limits_{N\rightarrow\infty}\bigg{(}\frac{1}{N}\sum_{i=1}^{N}\frac{\partial\varphi_{h}}{\partial
x_{h}}(\\{({\bm{N}}^{G,L}_{\ell,j})_{i*}\\}_{j=1}^{m},\\{(\widetilde{\bm{X}}_{j^{\prime}})_{i*}\\}_{j^{\prime}=1}^{\widetilde{m}})\bigg{)}^{\top},$
(C.154)
$\displaystyle\lim_{p_{h}\rightarrow\infty}\frac{({\bm{N}}^{G,R}_{k+1,h})^{\top}\vartheta_{h}({\bm{N}}^{G,R}_{k+1,h})}{p_{h}}=\lim_{N\rightarrow\infty}\frac{({\bm{N}}^{G,R}_{k+1,h})^{\top}{\bm{N}}^{G,R}_{\ell+1,h}}{p_{h}}\lim\limits_{N\rightarrow\infty}\bigg{(}\frac{1}{p_{h}}\sum_{i=1}^{p_{h}}\frac{\partial\vartheta_{h}}{\partial
x}({\bm{N}}^{G,R}_{k,h})\bigg{)}^{\top}.$ (C.155)

  5. 5.

For all $h\in[m]$ and $i,j\in[r_{h}]\times[r_{h}]$, we have

$\lim_{N\rightarrow\infty}\bigg{[}\frac{({\bm{N}}^{G,L}_{t,h})^{\top}{\bm{N}}^{G,L}_{t,h}}{N}\bigg{]}_{ij}<\infty\quad\mbox{and}\quad\lim_{p_{h}\rightarrow\infty}\bigg{[}\frac{({\bm{N}}^{G,R}_{t+1,h})^{\top}{\bm{N}}^{G,R}_{t+1,h}}{p_{h}}\bigg{]}_{ij}<\infty.$

  6. 6.

For all $0\leq s\leq t-1$ and $0\leq k\leq t$, we have positive definite
matrices $\boldsymbol{\mathsf{Z}}_{s,h}\in\mathbb{R}^{r_{h}\times r_{h}}$ and
$\boldsymbol{\mathsf{K}}_{k,h}\in\mathbb{R}^{r_{h}\times r_{h}}$ such that

$\lim_{N\rightarrow\infty}\frac{(\widebar{\bm{N}}^{G,L}_{s,h})^{\top}\bm{P}^{\perp}_{\boldsymbol{\widebar{\mathsf{N}}}^{G,L}_{t,h}}\widebar{\bm{N}}^{G,L}_{s,h}}{N}\succ\boldsymbol{\mathsf{Z}}_{s,h}\quad\mbox{and}\quad\lim_{p_{h}\rightarrow\infty}\frac{(\widebar{\bm{N}}^{G,R}_{s,h})^{\top}\bm{P}^{\perp}_{\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{t,h}}\widebar{\bm{N}}^{G,R}_{s,h}}{p_{h}}\succ\boldsymbol{\mathsf{K}}_{s,h}.$

We shall show Assertions (1)-(6) inductively. For $t=0$, (C.147) follows using
the techniques used to show Assertion (a) for $\bm{b}^{0}$ in [26]. Observing
that $\boldsymbol{\mathrm{N}}_{L,0,h}=\bm{0}$ and
$\bm{P}^{\perp}_{\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{0,h}}\widebar{\bm{N}}^{G,R}_{0,h}=\widebar{\bm{N}}^{G,R}_{0,h}$,
from (C.147) for $t=0$, we can conclude that for all $h\in[m]$, for all
$i\in[N]$

$[{\bm{N}}^{G,L}_{0,h}]_{i*}=\left(\frac{(\widebar{\bm{N}}^{G,R}_{0,h})^{\top}\widebar{\bm{N}}^{G,R}_{0,h}}{N}\right)^{1/2}Z,$

where $Z\sim N_{r_{h}}(0,\bm{I}_{r_{h}})$. Furthermore, the set of random
vectors $\\{[\widebar{\bm{N}}^{G,L}_{0,h}]_{i*}:i\in[N]\\}$ are independent.
From the definition (C.76), we can conclude that for all $N\in\mathbb{N}$

$\frac{(\widebar{\bm{N}}^{G,R}_{0,h})^{\top}\widebar{\bm{N}}^{G,R}_{0,h}}{N}=\bm{T}^{G,L}_{t,0}.$

Therefore, using Proposition 4.1 and the Strong Law of Large Numbers we can
conclude (2). The Assertions (3)-(6) for $\\{{\bm{N}}^{G,L}_{0,h}:h\in[m]\\}$
follows by retracing the techniques used to prove Lemma E.1 of [26]. We omit
the details to avoid the repetition of similar arguments. Furthermore,
Assertion (1) for $\\{{\bm{N}}^{G,R}_{1,h}:h\in[m]\\}$ follows using the
techniques used to prove Lemma E.1 of [26]. Proceeding as in the proof of
(3.36) of [26], we can show that

${\bm{N}}^{G,R}_{1,h}|_{\mathcal{G}_{1,0}}=\widetilde{\bm{G}}_{h}\widebar{\bm{N}}^{G,L}_{0,h}+o(1)\widebar{\bm{N}}^{G,R}_{0,h}.$

Since $\frac{|\mathcal{F}_{R,h}|}{p_{h}}\rightarrow\lambda_{R,h}\in(0,1]$,
using (4.6) and Cauchy Schwarz inequality we can get constants $C_{h}>0$
(possibly depending on $\lambda_{R,h}$) such that

$\displaystyle\lim_{p_{h}\rightarrow\infty}\Bigg{|}\frac{1}{|\mathcal{F}_{R,h}|}\sum_{i\in\mathcal{F}_{R,h}}\phi_{h}\left((\widetilde{\bm{G}}_{h}\widebar{\bm{N}}^{G,L}_{0,h}+o(1)\widebar{\bm{N}}^{G,R}_{0,h})_{i*},({\bm{V}}^{\mathrm{pc}}_{0,h})_{i*},(\bm{V}_{h})_{i*}\right)$
(C.156) $\displaystyle\hskip
70.0001pt-\frac{1}{|\mathcal{F}_{R,h}|}\sum_{i\in\mathcal{F}_{R,h}}\phi_{h}\left((\widetilde{\bm{G}}_{h}\widebar{\bm{N}}^{G,L}_{0,h})_{i*},({\bm{V}}^{\mathrm{pc}}_{0,h})_{i*},(\bm{V}_{h})_{i*}\right)\Bigg{|}$
(C.157) $\displaystyle\leq
C_{h}\left(1+\frac{\|\widetilde{\bm{G}}_{h}\widebar{\bm{N}}^{G,L}_{0,h}\|_{F}}{\sqrt{N}}+\frac{\|\widebar{\bm{N}}^{G,R}_{0,h}\|_{F}}{\sqrt{N}}+\frac{\|{\bm{V}}^{\mathrm{pc}}_{0,h}\|_{F}}{\sqrt{N}}+\frac{\|\bm{V}_{h}\|_{F}}{\sqrt{N}}\right)\times\frac{\|\widebar{\bm{N}}^{G,R}_{0,h}\|_{F}}{\sqrt{N}}\times
o(1)\xrightarrow{a.s}0.$ (C.158)

Consequently, in (C.156), given $\mathcal{G}_{1,0}$, we can replace
${\bm{N}}^{G,R}_{1,h}$ by $\widetilde{\bm{N}}^{G,R}_{1,h}$ for all $h\in[m]$
defined as follows

$\widetilde{\bm{N}}^{G,R}_{1,h}|_{\mathcal{G}_{1,0}}\overset{d}{=}\widetilde{\bm{G}}_{h}\widebar{\bm{N}}^{G,L}_{0,h}.$

Consequently, using Theorem 3 of [4], Proposition 4.1 and retracing the steps
of the proof of Lemma E.1 of [26], we can show (C.156). Again, one can show
Assertions (3)-(6) for ${\bm{N}}^{G,R}_{1,h}$ by retracing the steps of the
proof of Lemma E.1 of [26]. Now, let us assume that Assertions (1)-(6) hold
for $t=0,\ldots,s-1$. We shall show that they also hold for $t=s$. In fact, we
shall only show Assertion (2) for $t=s$, as the proofs of all other assertions
are similar to the proof of Lemma E.1 of [26]. To show the Assertion 2 for
${\bm{N}}^{G,L}_{s,h}$, observe that, since
$\frac{|\mathcal{F}_{L}|}{N}\rightarrow\lambda_{L}$, using Cauchy-Schwarz
inequality and (4.1), we have a constant $C>0$ (possibly depending on
$\lambda_{L}$), such that

$\displaystyle\lim_{N\rightarrow\infty}\Bigg{|}\frac{1}{|\mathcal{F}_{L}|}\sum_{i\in\mathcal{F}_{L}}\psi\Bigg{(}\\{(\boldsymbol{\mathrm{N}}_{L,s,h}\bm{\Xi}^{R}_{s,h}+\widetilde{\bm{G}}_{h}\bm{P}^{\perp}_{\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{s,h}}\widebar{\bm{N}}^{G,R}_{s,h}+\widetilde{\boldsymbol{\mathrm{N}}}_{R,s,h}\,o(1))_{i*}\\}_{h=1}^{m},\\{({\bm{U}}^{\mathrm{pc}}_{0,h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{X}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}},$
(C.159) $\displaystyle\hskip
180.00027pt\\{(\bm{U}_{h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{U}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}}\Bigg{)}$
$\displaystyle\hskip
40.00006pt-\frac{1}{|\mathcal{F}_{L}|}\sum_{i\in\mathcal{F}_{L}}\psi\Bigg{(}\\{(\boldsymbol{\mathrm{N}}_{L,s,h}\bm{\Xi}^{R}_{s,h}+\widetilde{\bm{G}}_{h}\bm{P}^{\perp}_{\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{s,h}}\widebar{\bm{N}}^{G,R}_{s,h})_{i*}\\}_{h=1}^{m},\\{({\bm{U}}^{\mathrm{pc}}_{0,h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{X}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}},$
(C.160) $\displaystyle\hskip
180.00027pt\\{(\bm{U}_{h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{U}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}}\Bigg{)}\Bigg{|},$
$\displaystyle\leq
C\left(1+\sum_{h=1}^{m}\frac{\left\|\boldsymbol{\mathrm{N}}_{L,s,h}\bm{\Xi}^{R}_{t,h}+\widetilde{\bm{G}}_{h}\bm{P}^{\perp}_{\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{s,h}}\widebar{\bm{N}}^{G,R}_{s,h}\right\|_{F}}{\sqrt{N}}+\sum_{h=1}^{m}\frac{\|\widetilde{\boldsymbol{\mathrm{N}}}_{R,s,h}\|_{F}}{\sqrt{N}}+\sum_{h=1}^{m}\frac{\|\bm{U}_{h}\|_{F}}{\sqrt{N}}\right.$
(C.161) $\displaystyle\left.\hskip
60.00009pt+\sum_{h=1}^{m}\frac{\|{\bm{U}}^{\mathrm{pc}}_{0,h}\|_{F}}{\sqrt{N}}+\sum_{\ell=1}^{\widetilde{m}}\frac{\|\widetilde{\bm{X}}_{\ell}\|_{F}}{\sqrt{N}}+\sum_{\ell=1}^{\widetilde{m}}\frac{\|\widetilde{\bm{U}}_{\ell}\|_{F}}{\sqrt{N}}\right)\times\left(\sum_{h=1}^{m}\frac{\|\widetilde{\boldsymbol{\mathrm{N}}}_{R,s,h}\|_{F}}{\sqrt{N}}\right)\times~{}o(1).$
(C.162)

Furthermore, from induction hypotheses

$\displaystyle\limsup_{N\rightarrow\infty}\sum_{h=1}^{m}\frac{\left\|\boldsymbol{\mathrm{N}}_{L,s,h}\bm{\Xi}^{R}_{t,h}+\widetilde{\bm{G}}_{h}\bm{P}^{\perp}_{\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{s,h}}\widebar{\bm{N}}^{G,R}_{s,h}\right\|_{F}}{\sqrt{N}}<\infty,\quad\limsup_{N\rightarrow\infty}\sum_{h=1}^{m}\frac{\|{\bm{U}}^{\mathrm{pc}}_{0,h}\|_{F}}{\sqrt{N}}<\infty$
(C.163)
$\displaystyle\limsup_{N\rightarrow\infty}\sum_{h=1}^{m}\frac{\|\widetilde{\boldsymbol{\mathrm{N}}}_{R,s,h}\|_{F}}{\sqrt{N}}<\infty,\quad\limsup_{N\rightarrow\infty}\sum_{\ell=1}^{\widetilde{m}}\frac{\|\widetilde{\bm{X}}_{\ell}\|_{F}}{\sqrt{N}}<\infty$
(C.164)
$\displaystyle\mbox{and}\quad\limsup_{N\rightarrow\infty}\sum_{\ell=1}^{\widetilde{m}}\frac{\|\widetilde{\bm{U}}_{\ell}\|_{F}}{\sqrt{N}}<\infty.$
(C.165)

Therefore, we can conclude that

$\displaystyle\lim_{N\rightarrow\infty}\Bigg{|}\frac{1}{|\mathcal{F}_{L}|}\sum_{i\in\mathcal{F}_{L}}\psi\Bigg{(}\\{(\boldsymbol{\mathrm{N}}_{L,s,h}\bm{\Xi}^{R}_{s,h}+\widetilde{\bm{G}}_{h}\bm{P}^{\perp}_{\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{s,h}}\widebar{\bm{N}}^{G,R}_{s,h}+\widetilde{\boldsymbol{\mathrm{N}}}_{R,s,h}\,o(1))_{i*}\\}_{h=1}^{m},\\{({\bm{U}}^{\mathrm{pc}}_{0,h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{X}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}},$
(C.166) $\displaystyle\hskip
180.00027pt\\{(\bm{U}_{h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{U}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}}\Bigg{)}$
$\displaystyle\hskip
40.00006pt-\frac{1}{|\mathcal{F}_{L}|}\sum_{i\in\mathcal{F}_{L}}\psi\Bigg{(}\\{(\boldsymbol{\mathrm{N}}_{L,s,h}\bm{\Xi}^{R}_{s,h}+\widetilde{\bm{G}}_{h}\bm{P}^{\perp}_{\boldsymbol{\widebar{\mathsf{N}}}^{G,R}_{s,h}}\widebar{\bm{N}}^{G,R}_{s,h})_{i*}\\}_{h=1}^{m},\\{({\bm{U}}^{\mathrm{pc}}_{0,h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{X}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}},$
(C.167) $\displaystyle\hskip
180.00027pt\\{(\bm{U}_{h})_{i*}\\}_{h=1}^{m},\\{(\widetilde{\bm{U}}_{\ell})_{i*}\\}_{\ell=1}^{\widetilde{m}}\Bigg{)}\Bigg{|}=0.$
(C.168)

Using the above observation, we can conclude (2) by using Theorem 3 of [4] and
retracing the steps used to prove Lemma 5 of [6] and Lemma E.1 of [26].
Furthermore, (C.156) can also be established using similar techniques. We omit
the details for the sake of brevity.

##  Appendix D Proofs of the Asymptotics of the AMP with Estimated Priors

Using Lemma B.4 in Zhong et al., [47] and Lemma 4.2, we have the following
version of Corollary B.5 of Zhong et al., [47].

######  Lemma D.1.

For $x=(x_{1},\ldots,x_{m})\in\mathbb{R}^{r}$ and
$\widetilde{x}=(\widetilde{x}_{1},\ldots,\widetilde{x}_{\widetilde{m}})\in\mathbb{R}^{\widetilde{r}}$

$\displaystyle\sup_{(x,\widetilde{x})}\left|f^{L}_{N}\left(x,\widetilde{x};\widehat{\mu}\right)-f^{L}\left(x,\widetilde{x};\mu\right)\right|\rightarrow
0.$ (D.1)

Similarly, for all $h\in[m]$,

$\displaystyle\sup_{y_{h}\in\mathbb{R}^{r_{h}}}\left|f^{R}_{h,N}(y_{h};\widehat{\nu}_{h})-f^{R}(y_{h};\nu_{h})\right|\rightarrow
0,$ (D.2)

where $f^{L}_{N},f^{L},f^{R}_{h,N}$ and $f^{R}_{h}$ are defined in Lemma B.1.

###  D.1 Proof of Theorem 4.1

We shall prove Theorem 4.1 using the following lemmas characterizing the
smoothness of the estimated denoisers. Let us consider the denoisers
$u^{\circ}_{t,h},v^{\circ}_{t,h}$ defined using the true priors $\mu$ and
$\nu_{h}$; the state evolution matrices
$\\{\bm{S}^{L,\circ}_{t,h},\bm{S}^{R,\circ}_{t,h},\bm{\Sigma}^{L,\circ}_{t,h},\bm{\Sigma}^{R,\circ}_{t,h}:t\geq
0,\,h\in[m]\\}$ and $\\{\bm{L}_{\ell}:\ell\in[\widetilde{m}]\\}$. Also
consider the approximate denoisers constructed using the Empirical Bayes
estimates of the priors $\widehat{\mu}$ and $\widehat{\nu}_{h}$, namely,
defined by (2.14), (2.16), the state evolution matrices
$\\{\widehat{\bm{S}}^{L}_{t,h},\widehat{\bm{S}}^{R}_{t,h},\widehat{\bm{\Sigma}}^{L}_{t,h},\widehat{\bm{\Sigma}}^{R}_{t,h}:t\geq
0\\}$ defined by (2.19) and the matrices
$\\{\widehat{\bm{L}}_{\ell}:\ell\in[\widetilde{m}]\\}$ defined in (2.11).

######  Lemma D.2.

For all matrices $\bm{F}_{h}\in\mathbb{R}^{N\times r_{h}}$,
$\widetilde{\bm{X}}_{\ell}\in\mathbb{R}^{N\times{\widetilde{r}}_{\ell}}$ and
$\bm{G}_{h}\in\mathbb{R}^{p_{h}\times r_{h}}$ where $h\in[m]$ and
$\ell\in[\widetilde{m}]$, as $N\rightarrow\infty$, we have

$\displaystyle\frac{1}{N}\|u_{t,h}(\bm{F}_{1},\ldots,\bm{F}_{m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}};\widehat{\mu})-u^{\circ}_{t,h}(\bm{F}_{1},\ldots,\bm{F}_{m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}};\mu)\|^{2}_{F}\xrightarrow{a.s.}0,$
(D.3)

and

$\displaystyle\frac{1}{p_{h}}\|v_{t,h}(\bm{G}_{h};\widehat{\nu}_{h})-v^{\circ}_{t,h}(\bm{G}_{h};\nu_{h})\|^{2}_{F}\xrightarrow{a.s.}0.$
(D.4)

In the above equations, for all $h\in[m]$ the functions
$u_{t,h}:\mathbb{R}^{r+\widetilde{r}}\rightarrow\mathbb{R}^{r_{h}}$ and
$v_{t,h}:\mathbb{R}^{r_{h}}\rightarrow\mathbb{R}^{r_{h}}$ are defined in
(2.18) and the functions
$u^{\circ}_{t,h}:\mathbb{R}^{r+\widetilde{r}}\rightarrow\mathbb{R}^{r_{h}}$
and $v^{\circ}_{t,h}:\mathbb{R}^{r_{h}}\rightarrow\mathbb{R}^{r_{h}}$ are
defined in (4.12). Similarly, for the functions
$\widetilde{u}_{t,h}:\mathbb{R}^{r+\widetilde{r}}\rightarrow\mathbb{R}^{\widetilde{r}_{\ell}}$
defined in (2.22) and
$\widetilde{u}^{\circ}_{t,h}:\mathbb{R}^{r+\widetilde{r}}\rightarrow\mathbb{R}^{\widetilde{r}_{\ell}}$
defined in (4.12), as $N\rightarrow\infty$

$\displaystyle\frac{1}{N}\|\widetilde{u}_{t,h}(\bm{F}_{1},\ldots,\bm{F}_{m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}};\widehat{\mu})-\widetilde{u}^{\circ}_{t,h}(\bm{F}_{1},\ldots,\bm{F}_{m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}};\mu)\|^{2}_{F}\xrightarrow{a.s.}0.$
(D.5)

Next, we consider the following lemma characterizing the smoothness of the
derivatives of the estimated denoisers.

######  Lemma D.3.

Consider the same set-up as Lemma D.2. Then as $N\rightarrow\infty$

$\displaystyle\frac{1}{N}\sum_{i=1}^{N}\|\bm{J}^{L}_{t,h}((\bm{F}_{1})_{i*},\ldots,(\bm{F}_{m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\widehat{\mu})$
(D.6) $\displaystyle\hskip
80.00012pt-\bm{J}^{L,\circ}_{t,h}((\bm{F}_{1})_{i*},\ldots,(\bm{F}_{m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\mu)\|_{\mathrm{op}}\xrightarrow{a.s.}0,$
(D.7)

and

$\frac{1}{p_{h}}\sum_{j=1}^{p_{h}}\|\bm{J}^{R}_{t,h}((\bm{G}_{h})_{i*};\widehat{\nu}_{h})-\bm{J}^{R,\circ}_{t,h}((\bm{G}_{h})_{i*};\nu_{h})\|_{\mathrm{op}}\xrightarrow{a.s.}0,$

for all matrices $\bm{F}_{h}\in\mathbb{R}^{N\times r_{h}}$,
$\widetilde{\bm{X}}_{\ell}\in\mathbb{R}^{N\times{\widetilde{r}}_{\ell}}$ and
$\bm{G}_{h}\in\mathbb{R}^{p_{h}\times r_{h}}$ where $h\in[m]$ and
$\ell\in[\widetilde{m}]$.

###### Proof of Theorem 4.1.

We shall prove the first two parts of the theorem using induction. Let us
formulate two sets of induction hypotheses $\mathcal{H}_{t}$ and
$\bar{\mathcal{H}}_{t}$ for $t\in\mathbb{N}\cup\\{0\\}$, where
$\mathcal{H}_{t}$ implies:

  1. 1.

$\widehat{\bm{S}}^{R}_{t,h}\rightarrow\bm{S}^{R,\circ}_{t,h}$ and
$\widehat{\bm{\Sigma}}^{R}_{t,h}\rightarrow\bm{\Sigma}^{R,\circ}_{t,h}$ as
$N\rightarrow\infty$,

  2. 2.

As $N\rightarrow\infty$,
$\frac{1}{N}\|\widebar{\bm{U}}_{t-1,h}-\widebar{\bm{U}}^{\circ}_{t-1,h}\|^{2}_{F}\xrightarrow{a.s.}0$,

  3. 3.

As $N\rightarrow\infty$,
$\frac{1}{N}\|\widetilde{\bm{U}}_{t-1,\ell}-\widetilde{\bm{U}}^{\circ}_{t-1,\ell}\|^{2}_{F}\xrightarrow{a.s.}0$,
for $t\geq 1$,

  4. 4.

As $N\rightarrow\infty$,
$\frac{1}{p_{h}}\|\bm{V}_{t,h}-\bm{V}^{\circ}_{t,h}\|^{2}_{F}\xrightarrow{a.s.}0$,
for all $h\in[m],\;\ell\in[\widetilde{m}]$,

and $\bar{\mathcal{H}}_{t}$ implies:

  1. 1.

$\widehat{\bm{S}}^{L}_{t,h}\rightarrow\bm{S}^{L,\circ}_{t,h}$ and
$\widehat{\bm{\Sigma}}^{L}_{t,h}\rightarrow\bm{\Sigma}^{L,\circ}_{t,h}$ as
$N\rightarrow\infty$.

  2. 2.

As $p_{h}\rightarrow\infty$,
$\frac{1}{p_{h}}\|\widebar{\bm{V}}_{t,h}-\widebar{\bm{V}}^{\circ}_{t,h}\|^{2}_{F}\xrightarrow{a.s.}0$,

  3. 3.

As $p_{h}\rightarrow\infty$,
$\frac{1}{N}\|\bm{U}_{t,h}-\bm{U}^{\circ}_{t,h}\|^{2}_{F}\xrightarrow{a.s.}0$,
for all $h\in[m]$.

##### Step 1: Initialization $\mathcal{H}_{0}$.

By (2.7), (2.9) and the results from [5], the first assertion holds. For the
second assertion observe the following:

$\frac{1}{N}\|\widebar{\bm{U}}_{-1,h}-\widebar{\bm{U}}^{\circ}_{-1,h}\|^{2}_{F}\leq\frac{1}{N}\|{\bm{U}}^{\mathrm{pc}}_{0,h}\|^{2}_{F}\;\|(\widehat{\bm{\Sigma}}^{R,\mathrm{pc}}_{0,h})^{1/2}-(\bm{\Sigma}^{R,\mathrm{pc}}_{0,h})^{1/2}\|^{2}_{\mathrm{op}}.$

By Proposition 4.1,
$\limsup\limits_{N\rightarrow\infty}\frac{1}{N}\|{\bm{U}}^{\mathrm{pc}}_{0,h}\|^{2}_{F}<\infty$
almost surely. Finally, since the plug-in estimator of
$\widehat{\bm{\Sigma}}^{R,\mathrm{pc}}_{0,h}$ converges to
$\bm{\Sigma}^{R,\mathrm{pc}}_{0,h}$ as $N\rightarrow\infty$, the second
assertion holds. Further, as both algorithms, given by (2.17) and (4.13), have
the same initialization for $\bm{V}_{0,h}$ for all $h\in[m]$ and
$\ell\in[\widetilde{m}]$, the fourth assertions also hold. The third assertion
is not applicable for $\mathcal{H}_{0}$.

##### Step 2: $\mathcal{H}_{t}\implies\bar{\mathcal{H}}_{t}$.

This inductive step follows using the arguments of the proof of Theorem 5.4 in
Zhong et al., [47].

##### Step 3: $\bar{\mathcal{H}}_{t}\implies\mathcal{H}_{t+1}$.

First, let us consider the second assertion. Observe that

$\displaystyle\frac{1}{N}\|\widebar{\bm{U}}_{t,h}-\widebar{\bm{U}}^{\circ}_{t,h}\|^{2}_{F}$
(D.8)
$\displaystyle=\frac{1}{N}\|u_{t,h}(\bm{U}_{t,1},\ldots,\bm{U}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}};\widehat{\mu})-u^{\circ}_{t,h}(\bm{U}^{\circ}_{t,1},\ldots,\bm{U}^{\circ}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}};\mu)\|^{2}_{F}$
(D.9)
$\displaystyle=\frac{1}{N}\sum_{i=1}^{N}\|u_{t,h}((\bm{U}_{t,1})_{i*},\ldots,(\bm{U}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\widehat{\mu})$
(D.10) $\displaystyle\hskip
100.00015pt-u^{\circ}_{t,h}((\bm{U}^{\circ}_{t,1})_{i*},\ldots,(\bm{U}^{\circ}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\mu)\|^{2}_{2}$
(D.11)
$\displaystyle\leq\frac{2}{N}\sum_{i=1}^{N}\|u_{t,h}((\bm{U}_{t,1})_{i*},\ldots,(\bm{U}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\widehat{\mu})$
(D.12) $\displaystyle\hskip
100.00015pt-u^{\circ}_{t,h}((\bm{U}_{t,1})_{i*},\ldots,(\bm{U}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\mu)\|^{2}_{2}$
(D.13) $\displaystyle\hskip
10.00002pt+\frac{2}{N}\sum_{i=1}^{N}\|u^{\circ}_{t,h}((\bm{U}_{t,1})_{i*},\ldots,(\bm{U}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\mu)$
(D.14) $\displaystyle\hskip
100.00015pt-u^{\circ}_{t,h}((\bm{U}^{\circ}_{t,1})_{i*},\ldots,(\bm{U}^{\circ}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\mu)\|^{2}_{2}.$
(D.15)

Let us consider the first expression on the right-hand side of the above
equation. By Lemma D.2 we have:

$\displaystyle\frac{2}{N}\sum_{i=1}^{N}\|u_{t,h}((\bm{U}_{t,1})_{i*},\ldots,(\bm{U}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\widehat{\mu})$
(D.16) $\displaystyle\hskip
100.00015pt-u^{\circ}_{t,h}((\bm{U}_{t,1})_{i*},\ldots,(\bm{U}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\mu)\|^{2}_{2}\xrightarrow{a.s.}0.$
(D.17)

Next, for second expression on the right-hand side of (D.8), we use the
Lipschitz property of the denoisers implied by Assumption 4.1, to conclude
that for all $h\in[m]$

$\displaystyle\frac{2}{N}\sum_{i=1}^{N}\|u^{\circ}_{t,h}((\bm{U}_{t,1})_{i*},\ldots,(\bm{U}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\mu)$
(D.18) $\displaystyle\hskip
100.00015pt-u^{\circ}_{t,h}((\bm{U}^{\circ}_{t,1})_{i*},\ldots,(\bm{U}^{\circ}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\mu)\|^{2}_{2}$
(D.19)
$\displaystyle\leq\frac{C_{1}}{N}\sum_{h=1}^{m}\|\bm{U}_{t,h}-\bm{U}^{\circ}_{t,h}\|^{2}_{F}\xrightarrow{a.s.}0,\quad\mbox{as
$N\rightarrow\infty$}.$ (D.20)

Here $C_{1}>0$ is an absolute constant and the last relation follows using
$\bar{\mathcal{H}}_{t}(3)$. For the third assertion, observe that

$\displaystyle\frac{1}{N}\|\widetilde{\bm{U}}_{t,h}-\widetilde{\bm{U}}^{\circ}_{t,h}\|^{2}_{F}$
(D.21)
$\displaystyle=\frac{1}{N}\|\widetilde{u}_{t,h}(\bm{U}_{t,1},\ldots,\bm{U}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}};\widehat{\mu})-\widetilde{u}^{\circ}_{t,h}(\bm{U}^{\circ}_{t,1},\ldots,\bm{U}^{\circ}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}};\mu)\|^{2}_{F}$
(D.22)
$\displaystyle=\frac{1}{N}\sum_{i=1}^{N}\|\widetilde{u}_{t,h}((\bm{U}_{t,1})_{i*},\ldots,(\bm{U}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\widehat{\mu})$
(D.23) $\displaystyle\hskip
100.00015pt-\widetilde{u}^{\circ}_{t,h}((\bm{U}^{\circ}_{t,1})_{i*},\ldots,(\bm{U}^{\circ}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\mu)\|^{2}_{2}.$
(D.24)

The right-hand side of (D.21) converges to $0$ by (D.5), Lemma D.2 and
arguments as before. Finally, consider the fourth assertion. Define

$\bm{J}^{L}_{t,h}\equiv\bm{J}^{L}_{t,h}(\bm{U}_{t,1},\ldots,\bm{U}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}};\widehat{\mu})$

and

$\bm{J}^{L,\circ}_{t,h}\equiv\bm{J}^{L,\circ}_{t,h}(\bm{U}^{\circ}_{t,1},\ldots,\bm{U}^{\circ}_{t,m},\widetilde{\bm{X}}_{1},\ldots,\widetilde{\bm{X}}_{\widetilde{m}};\mu).$

We can get a constant $C_{2}>0$ such that

$\displaystyle\frac{1}{p_{h}}\|\bm{V}_{t+1,h}-\bm{V}^{\circ}_{t+1,h}\|^{2}_{F}$
$\displaystyle\leq\frac{C_{2}}{p_{h}}\|\bm{X}^{\top}_{h}\widebar{\bm{U}}_{t,h}-\bm{X}^{\top}_{h}\widebar{\bm{U}}^{\circ}_{t,h}\|^{2}_{F}+C_{2}\frac{\|\widebar{\bm{V}}_{t,h}\|^{2}_{F}}{p_{h}}\|\bm{J}^{L}_{t,h}-\bm{J}^{L,\circ}_{t,h}\|^{2}_{\mathrm{op}}$
(D.25) $\displaystyle\hskip
30.00005pt+\frac{C_{2}}{p_{h}}\|\widebar{\bm{V}}_{t,h}-\widebar{\bm{V}}^{\circ}_{t,h}\|^{2}_{F}\|\bm{J}^{L,\circ}_{t,h}\|^{2}_{\mathrm{op}}$
(D.26)
$\displaystyle\leq\frac{C_{2}}{p_{h}}\|\bm{X}_{h}\|^{2}_{\mathrm{op}}\|\widebar{\bm{U}}_{t,h}-\widebar{\bm{U}}^{\circ}_{t,h}\|^{2}_{F}+C_{2}\frac{\|\widebar{\bm{V}}_{t,h}\|^{2}_{F}}{p_{h}}\|\bm{J}^{L}_{t,h}-\bm{J}^{L,\circ}_{t,h}\|^{2}_{\mathrm{op}}$
(D.27) $\displaystyle\hskip
30.00005pt+\frac{C_{2}}{p_{h}}\|\widebar{\bm{V}}_{t,h}-\widebar{\bm{V}}^{\circ}_{t,h}\|^{2}_{F}\|\bm{J}^{L,\circ}_{t,h}\|^{2}_{\mathrm{op}}.$
(D.28)

By Corollary 5.35 of Vershynin, [45] we have
$\limsup_{N\rightarrow\infty}\|\bm{X}_{h}\|_{\mathrm{op}}<\infty$ almost
surely. By $\mathcal{\bar{H}}_{t}(2)$, $\mathcal{H}_{t}(2)$ and Theorem 4.1,
we can conclude that
$\limsup_{p_{h}\rightarrow\infty}\frac{\|\widebar{\bm{V}}_{t,h}\|^{2}_{F}}{p_{h}}<\infty$,
almost surely. Further, observe that

$\displaystyle\bm{J}^{L}_{t,h}-\bm{J}^{L,\circ}_{t,h}$ (D.29)
$\displaystyle=\frac{1}{N}\sum_{i=1}^{N}\bigg{[}\bm{J}^{L}_{t,h}((\bm{U}_{t,1})_{i*},\ldots,(\bm{U}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\widehat{\mu})$
(D.30) $\displaystyle\hskip
80.00012pt-\bm{J}^{L,\circ}_{t,h}((\bm{U}^{\circ}_{t,1})_{i*},\ldots,(\bm{U}^{\circ}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\mu)\bigg{]}$
(D.31)
$\displaystyle\leq\frac{1}{N}\sum_{i=1}^{N}\bigg{[}\bm{J}^{L}_{t,h}((\bm{U}_{t,1})_{i*},\ldots,(\bm{U}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\widehat{\mu})$
(D.32) $\displaystyle\hskip
80.00012pt-\bm{J}^{L,\circ}_{t,h}((\bm{U}_{t,1})_{i*},\ldots,(\bm{U}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\mu)\bigg{]}$
(D.33)
$\displaystyle+\frac{1}{N}\sum_{i=1}^{N}\bigg{[}\bm{J}^{L,\circ}_{t,h}((\bm{U}_{t,1})_{i*},\ldots,(\bm{U}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\mu)$
(D.34) $\displaystyle\hskip
80.00012pt-\bm{J}^{L,\circ}_{t,h}((\bm{U}^{\circ}_{t,1})_{i*},\ldots,(\bm{U}^{\circ}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\mu)\bigg{]}.$
(D.35)

For the expression in (D.32), using Lemma D.3 can conclude that

$\displaystyle\Bigg{\|}\frac{1}{N}\sum_{i=1}^{N}\bigg{[}\bm{J}^{L}_{t,h}((\bm{U}_{t,1})_{i*},\ldots,(\bm{U}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\widehat{\mu})$
(D.36) $\displaystyle\hskip
80.00012pt-\bm{J}^{L,\circ}_{t,h}((\bm{U}_{t,1})_{i*},\ldots,(\bm{U}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\mu)\bigg{]}\Bigg{\|}_{\mathrm{op}}$
(D.37)
$\displaystyle\leq\frac{1}{N}\sum_{i=1}^{N}\bigg{\|}\bm{J}^{L}_{t,h}((\bm{U}_{t,1})_{i*},\ldots,(\bm{U}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\widehat{\mu})$
(D.38) $\displaystyle\hskip
80.00012pt-\bm{J}^{L,\circ}_{t,h}((\bm{U}_{t,1})_{i*},\ldots,(\bm{U}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\mu)\bigg{\|}_{\mathrm{op}}$
(D.39) $\displaystyle\hskip 20.00003pt\xrightarrow{a.s.}0,\quad\mbox{as
$N\rightarrow\infty$.}$ (D.40)

To analyze the expression in (D.34), we first express the function
$u^{\circ}_{t,h}(x_{1},\ldots,x_{m},\widetilde{x}_{1},\ldots,\widetilde{x}_{\widetilde{m}};\mu)$
using Tweedie’s formula as follows:

$\displaystyle
u^{\circ}_{t,h}(x_{1},\ldots,x_{m},\widetilde{x}_{1},\ldots,\widetilde{x}_{\widetilde{m}};\mu)=x_{h}+\frac{\nabla_{h}f^{L}_{t}(x_{1},\ldots,x_{m},\widetilde{x}_{1},\ldots,\widetilde{x}_{\widetilde{m}};\mu)}{f^{L}_{t}(x_{1},\ldots,x_{m},\widetilde{x}_{1},\ldots,\widetilde{x}_{\widetilde{m}};\mu)},$
(D.41)

where $\nabla_{h}$ refers to the gradient of a function
$f:\mathbb{R}^{r+\widetilde{r}}\rightarrow\mathbb{R}$ with respect to the
vector of argument $x_{h}$. The function
$f^{L}_{t}:\mathbb{R}^{r+\widetilde{r}}\rightarrow\mathbb{R}$ is defined as
follows:

$\displaystyle
f^{L}_{t}(x_{1},\ldots,x_{m},\widetilde{x}_{1},\ldots,\widetilde{x}_{\widetilde{m}};\mu)\equiv$
$\displaystyle\int_{u,\tilde{u}}\prod_{h=1}^{m}\phi_{r_{h}}((\bm{\Sigma}^{L,\circ}_{t,h})^{-1/2}(x_{h}-\bm{S}^{L,\circ}_{t,h}u_{h}))\times$
(D.42) $\displaystyle\hskip
60.00009pt\prod_{\ell=1}^{\widetilde{m}}\phi_{\widetilde{r}_{\ell}}(\widetilde{x}_{\ell}-\bm{L}_{\ell}\widetilde{u}_{\ell})d\mu(u_{1},\ldots,u_{m},\widetilde{u}_{1},\ldots,\widetilde{u}_{\ell}).$
(D.43)

By Assumption 4.1, since $u^{\circ}_{t,h}$ is uniformly Lipschitz, therefore
$\bm{J}^{L,\circ}_{t,h}$ is uniformly bounded by a constant $C_{L}$. Further,
for all $h\in[m]$ and $\ell\in[\widetilde{m}]$;
$x_{h}\in\mathbb{B}_{r_{h}}(B)$ and
$\widetilde{x}_{\ell}\in\mathbb{B}_{{\widetilde{r}}_{\ell}}(B)$, then there
exits a constant $c>0$, such that
$f_{t}(x_{1},\ldots,x_{m},\widetilde{x}_{1},\ldots,\widetilde{x}_{\widetilde{m}};\mu)>c$
which implies that the derivative of $\bm{J}^{L,\circ}_{t,h}$ with respect to
the variable $x_{h}$ is uniformly bounded by a constant $C_{B}>0$ for all
$x_{h}\in\mathbb{B}_{r_{h}}(B)$ and
$\widetilde{x}_{\ell}\in\mathbb{B}_{{\widetilde{r}}_{\ell}}(B)$. Hence, if
$x_{h}\in\mathbb{B}_{r_{h}}(B)$ and
$\widetilde{x}_{\ell}\in\mathbb{B}_{{\widetilde{r}}_{\ell}}(B)$ for all
$h\in[m]$ and $\ell\in[\widetilde{m}]$, then $\bm{J}^{L,\circ}_{t,h}$ is
Lipschitz continuous with Lipschitz constant $C_{B}$. Now, consider the term
in (D.34).

$\displaystyle\frac{1}{N}\sum_{i=1}^{N}\bigg{\|}\bm{J}^{L,\circ}_{t,h}((\bm{U}_{t,1})_{i*},\ldots,(\bm{U}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};{\mu})$
(D.44) $\displaystyle\hskip
80.00012pt-\bm{J}^{L,\circ}_{t,h}((\bm{U}^{\circ}_{t,1})_{i*},\ldots,(\bm{U}^{\circ}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\mu)\bigg{\|}_{\mathrm{op}}$
(D.45)
$\displaystyle\leq\frac{1}{N}\sum_{i=1}^{N}\bigg{\|}\bm{J}^{L,\circ}_{t,h}((\bm{U}_{t,1})_{i*},\ldots,(\bm{U}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};{\mu})$
(D.46) $\displaystyle\hskip
80.00012pt-\bm{J}^{L,\circ}_{t,h}((\bm{U}^{\circ}_{t,1})_{i*},\ldots,(\bm{U}^{\circ}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\mu)\bigg{\|}_{\mathrm{op}}$
(D.47) $\displaystyle\hskip
50.00008pt\times\prod_{h=1}^{m}\mathbb{I}(\|(\bm{U}_{t,h})_{i*}\|_{2}\leq
B)\times\prod_{\ell=1}^{n}\mathbb{I}(\|(\widetilde{\bm{X}}_{\ell})_{i*}\|_{2}\leq
B)\times\prod_{h=1}^{m}\mathbb{I}(\|(\bm{U}^{\circ}_{t,h})_{i*}\|_{2}\leq B)$
(D.48) $\displaystyle\hskip
20.00003pt+\frac{C_{L}}{N}\sum_{i=1}^{N}\sum_{h=1}^{m}\mathbb{I}(\|(\bm{U}_{t,h})_{i*}\|_{2}\geq
B)+\frac{C_{L}}{N}\sum_{i=1}^{N}\sum_{\ell=1}^{\widetilde{m}}\mathbb{I}(\|(\widetilde{\bm{X}}_{\ell})_{i*}\|_{2}\geq
B)$ (D.49) $\displaystyle\hskip
80.00012pt+\frac{C_{L}}{N}\sum_{i=1}^{N}\sum_{h=1}^{m}\mathbb{I}(\|(\bm{U}^{\circ}_{t,h})_{i*}\|_{2}\geq
B).$ (D.50)

Since $\|x\|^{2}_{2}\geq B^{2}\mathbb{I}(\|x\|_{2}\geq B)$, for all
$x\in\mathbb{R}^{r_{h}}$, the last two terms in the above display can be
bounded as follows:

$\displaystyle\frac{C_{L}}{N}\sum_{i=1}^{N}\sum_{h=1}^{m}\mathbb{I}(\|(\bm{U}_{t,h})_{i*}\|_{2}\geq
B)+\frac{C_{L}}{N}\sum_{i=1}^{N}\sum_{\ell=1}^{\widetilde{m}}\mathbb{I}(\|(\widetilde{\bm{X}}_{\ell})_{i*}\|_{2}\geq
B)+\frac{C_{L}}{N}\sum_{i=1}^{N}\sum_{h=1}^{m}\mathbb{I}(\|(\bm{U}^{\circ}_{t,h})_{i*}\|_{2}\geq
B)$ (D.51)
$\displaystyle\leq\frac{C_{L}}{NB^{2}}\sum_{h=1}^{m}\|\bm{U}_{t,h}\|^{2}_{F}+\frac{C_{L}}{NB^{2}}\sum_{h=1}^{m}\|\bm{U}^{\circ}_{t,h}\|^{2}_{F}+\frac{C_{L}}{NB^{2}}\sum_{\ell=1}^{\widetilde{m}}\|\widetilde{\bm{X}}_{\ell}\|^{2}_{F}.$
(D.52)

By definition of $\widetilde{\bm{X}}_{\ell}$, $\mathcal{\widebar{H}}_{t}(3)$
and Theorem C.1, we can conclude that, almost surely

$\limsup_{N\rightarrow\infty}\left\\{\frac{C_{L}}{N}\sum_{h=1}^{m}\|\bm{U}_{t,h}\|^{2}_{F}+\frac{C_{L}}{N}\sum_{h=1}^{m}\|\bm{U}^{\circ}_{t,h}\|^{2}_{F}+\frac{C_{L}}{N}\sum_{\ell=1}^{\widetilde{m}}\|\widetilde{\bm{X}}_{\ell}\|^{2}_{F}\right\\}<\infty.$

Hence, if we take $B\rightarrow\infty$, we get

$\displaystyle\limsup_{B\rightarrow\infty}\left\\{\frac{C_{L}}{N}\sum_{i=1}^{N}\sum_{h=1}^{m}\mathbb{I}(\|(\bm{U}_{t,h})_{i*}\|_{2}\geq
B)+\frac{C_{L}}{N}\sum_{i=1}^{N}\sum_{\ell=1}^{\widetilde{m}}\mathbb{I}(\|(\widetilde{\bm{X}}_{\ell})_{i*}\|_{2}\geq
B)\right.$ (D.53) $\displaystyle\hskip
50.00008pt\left.+\frac{C_{L}}{N}\sum_{i=1}^{N}\sum_{h=1}^{m}\mathbb{I}(\|(\bm{U}^{\circ}_{t,h})_{i*}\|_{2}\geq
B)\right\\}\overset{a.s.}{=}0$ (D.54)

To analyze the other term we use the Lipschitz property of the Jacobian
matrices of the denoisers when its arguments are confined to an Euclidean ball
of appropriate dimensions and radius $B>0$. That condition implies

$\displaystyle\frac{1}{N}\sum_{i=1}^{N}\bigg{\|}\bm{J}^{L,\circ}_{t,h}((\bm{U}_{t,1})_{i*},\ldots,(\bm{U}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};{\mu})$
(D.55) $\displaystyle\hskip
80.00012pt-\bm{J}^{L,\circ}_{t,h}((\bm{U}^{\circ}_{t,1})_{i*},\ldots,(\bm{U}^{\circ}_{t,m})_{i*},(\widetilde{\bm{X}}_{1})_{i*},\ldots,(\widetilde{\bm{X}}_{\widetilde{m}})_{i*};\mu)\bigg{\|}_{\mathrm{op}}$
(D.56) $\displaystyle\hskip
50.00008pt\times\prod_{h=1}^{m}\mathbb{I}(\|(\bm{U}_{t,h})_{i*}\|_{2}\leq
B)\times\prod_{\ell=1}^{n}\mathbb{I}(\|(\widetilde{\bm{X}}_{\ell})_{i*}\|_{2}\leq
B)\times\prod_{h=1}^{m}\mathbb{I}(\|(\bm{U}^{\circ}_{t,h})_{i*}\|_{2}\leq B)$
(D.57)
$\displaystyle\leq\frac{C_{B}}{\sqrt{N}}\sum_{h=1}^{m}\|\bm{U}_{t,h}-\bm{U}^{\circ}_{t,h}\|_{F}\xrightarrow{a.s.}0.$
(D.58)

Hence, we can conclude that
$\|\bm{J}^{L}_{t,h}-\bm{J}^{L,\circ}_{t,h}\|\xrightarrow{a.s.}0$, as
$N\rightarrow\infty$. Using similar arguments one can also show that
$\limsup_{N\rightarrow\infty}\|\bm{J}^{L,\circ}_{t,h}\|<\infty$, almost
surely. Combining (D.25), $\mathcal{H}_{t+1}(2)$ and $\bar{\mathcal{H}}_{t}$,
the third assertion of $\mathcal{H}_{t+1}$ follows.

To show the first assertion of $\mathcal{H}_{t+1}$, we observe that
$u^{\otimes 2}_{t,h}$ is a pseudo-Lipschitz function. Hence, the assertion
follows using the techniques described in the proof of Theorem 5.4 in Zhong et
al., [47].

Finally, since $\frac{|\mathcal{F}_{L}|}{N}\rightarrow\lambda_{L}\in(0,1]$ and
$\frac{|\mathcal{F}_{R,h}|}{p_{h}}\rightarrow\lambda_{R,h}\in(0,1]$ for all
$h\in[m]$, the third part of the theorem follows using Theorem C.1 and the
property of pseudo-Lipschitz functions (in particular, inequalities of form
(C.135)). ∎

###  D.2 Proof of Lemma D.1

We shall prove the statement (D.1). The proof of (D.2) is similar. Denote by
$\widehat{\bm{\Theta}}=\bigg{(}\\{\widehat{\bm{S}}^{L,\mathrm{pc}}_{0,h}\\}_{h=1}^{m},\newline
\\{\widehat{\bm{\Sigma}}^{L,\mathrm{pc}}_{0,h}\\}_{h=1}^{m},\\{\widehat{\bm{L}}_{\ell}\\}_{\ell=1}^{\widetilde{m}}\bigg{)}$
and
$\bm{\Theta}=\bigg{(}\\{\bm{S}^{L,\mathrm{pc}}_{0,h}\\}_{h=1}^{m},\\{\bm{\Sigma}^{L,\mathrm{pc}}_{0,h}\\}_{h=1}^{m},\\{\bm{L}_{\ell}\\}_{\ell=1}^{\widetilde{m}}\bigg{)}$.
Consider the functions

$\xi^{L}_{N}((x,\widetilde{x});(u,\widetilde{u});\widehat{\bm{\Theta}})=\prod_{h=1}^{m}\phi_{r_{h}}((\widehat{\bm{\Sigma}}^{L,\mathrm{pc}}_{0,h})^{-1/2}(x_{h}-\widehat{\bm{S}}^{L,\mathrm{pc}}_{0,h}u_{h}))\prod_{\ell=1}^{\widetilde{m}}\phi_{\widetilde{r}_{\ell}}(\widetilde{x}_{\ell}-\widehat{\bm{L}}_{\ell}\widetilde{u}_{\ell}),$

and

$\xi((x,\widetilde{x});(u,\widetilde{u});\bm{\Theta})=\prod_{h=1}^{m}\phi_{r_{h}}((\bm{\Sigma}^{L,\mathrm{pc}}_{0,h})^{-1/2}(x_{h}-\bm{S}^{L,\mathrm{pc}}_{0,h}u_{h}))\prod_{\ell=1}^{\widetilde{m}}\phi_{\widetilde{r}_{\ell}}(\widetilde{x}_{\ell}-\bm{L}_{\ell}\widetilde{u}_{\ell}).$

Since, by Lemma 4.1, $\widehat{\bm{\Theta}}\rightarrow\bm{\Theta}$ as
$N\rightarrow\infty$, both
$\xi^{L}_{N}((x,\widetilde{x});(u,\widetilde{u});\widehat{\bm{\Theta}})$ and
$\xi((x,\widetilde{x});(u,\widetilde{u});\bm{\Theta})$ and their derivatives
are uniformly bounded with respect to $(x,\widetilde{x})$, $(u,\widetilde{u})$
and $\widehat{\bm{\Theta}}$, when $\widehat{\bm{\Theta}}$ is contained in a
finite radius neighborhood of $\bm{\Theta}$.

Then, proceeding as in the proof of Lemma B.4 and Corollary B.5 of Zhong et
al., [47], the stated result follows.

###  D.3 Proof of Lemma D.2

We shall prove (D.3) and the proof of (D.4) shall follow similarly. Let us
consider the collection of matrices
$\bm{\Theta}=\bigg{(}\\{\bm{S}^{L,\mathrm{pc}}_{0,h}\\}_{h=1}^{m},\\{\bm{\Sigma}^{L,\mathrm{pc}}_{0,h}\\}_{h=1}^{m},\\{\bm{L}_{\ell}\\}_{\ell=1}^{\widetilde{m}}\bigg{)}$
and
$\widehat{\bm{\Theta}}=\bigg{(}\\{\widehat{\bm{S}}^{L,\mathrm{pc}}_{0,h}\\}_{h=1}^{m},\\{\widehat{\bm{\Sigma}}^{L,\mathrm{pc}}_{0,h}\\}_{=1}^{m},\\{\widehat{\bm{L}}_{\ell}\\}_{\ell=1}^{\widetilde{m}}\bigg{)}$.
By Tweedie’s formula, for $h\in[m]$, we can write the posterior mean denoisers
as

$u_{t,h}(x_{1},\ldots,x_{m},\widetilde{x}_{1},\ldots,\widetilde{x}_{\widetilde{m}};\widehat{\mu})=(\widehat{\bm{S}}^{L,\mathrm{pc}}_{0,h})^{-1}\bigg{(}\widehat{\bm{\Sigma}}^{L,\mathrm{pc}}_{0,h}\frac{\nabla_{x_{h}}f^{L}_{N}(x_{1},\ldots,x_{m},\widetilde{x}_{1},\ldots,\widetilde{x}_{\widetilde{m}};\widehat{\mu})^{\top}}{f^{L}_{N}(x_{1},\ldots,x_{m},\widetilde{x}_{1},\ldots,\widetilde{x}_{\widetilde{m}};\widehat{\mu})}+x_{h}\bigg{)},$

and

$u^{\circ}_{t,h}(x_{1},\ldots,x_{m},\widetilde{x}_{1},\ldots,\widetilde{x}_{\widetilde{m}};\mu)=(\bm{S}^{L,\mathrm{pc}}_{0,h})^{-1}\bigg{(}\bm{\Sigma}^{L,\mathrm{pc}}_{0,h}\frac{\nabla_{x_{h}}f^{L}(x_{1},\ldots,x_{m},\widetilde{x}_{1},\ldots,\widetilde{x}_{\widetilde{m}};\mu)^{\top}}{f^{L}(x_{1},\ldots,x_{m},\widetilde{x}_{1},\ldots,\widetilde{x}_{\widetilde{m}};\mu)}+x_{h}\bigg{)},$

where for any function $f$ defined on appropriate domain,
$\nabla_{x_{h}}f(x_{1},\ldots,x_{m},\widetilde{x}_{1},\ldots,\widetilde{x}_{\widetilde{m}};\widehat{\mu})$
denotes the gradient of the function $f$ with respect to $x_{h}$. In the above
equations the functions $f^{L}_{N}$ and $f^{L}$ are defined in (B.1) and
(B.4), respectively. By Lemma D.1, for $N\rightarrow\infty$, we have

$\displaystyle\sup_{(x,\widetilde{x})}\Bigg{|}f^{L}_{N}(x_{1},\ldots,x_{m},\widetilde{x}_{1},\ldots,\widetilde{x}_{\widetilde{m}};\widehat{\mu})-f^{L}(x_{1},\ldots,x_{m},\widetilde{x}_{1},\ldots,\widetilde{x}_{\widetilde{m}};\mu)\Bigg{|}\rightarrow
0.$ (D.59)

Using similar techniques as the proof of Lemma D.1, we can also show that for
$h\in[m]$:

$\displaystyle\sup_{(x,\widetilde{x})}\Bigg{\|}\nabla_{x_{h}}f^{L}_{N}(x_{1},\ldots,x_{m},\widetilde{x}_{1},\ldots,\widetilde{x}_{\widetilde{m}};\widehat{\mu})-\nabla_{x_{h}}f^{L}(x_{1},\ldots,x_{m},\widetilde{x}_{1},\ldots,\widetilde{x}_{\widetilde{m}};\mu)\Bigg{\|}_{\mathrm{op}}\rightarrow
0.$ (D.60)

Since, by Lemma 4.1, $\widehat{\bm{\Theta}}\rightarrow\bm{\Theta}$ as
$N\rightarrow\infty$, we have an open bounded neighborhood of $\bm{\Theta}$
such that as $N\rightarrow\infty$, the matrices in the collection
$\widehat{\bm{\Theta}}$ lie in it. Then using Assumption 4.1, we can verify
along the lines of Proposition B.1 of Zhong et al., [47] that there exists a
neighborhood of $(\bm{\Theta},\mu)$ (with respect to the topology of weak
convergence in $\mu$) where $u_{t,h}$ is uniformly Lipschitz in
$(x,\widetilde{x})$. Therefore, following the analysis of Corollary B.3 of
Zhong et al., [47], we can conclude (D.3). The proofs of (D.4) and (D.5)
follow using similar arguments.

###  D.4 Proof of Lemma D.3

The proof of this Lemma follows using the techniques used to prove Lemma D.2
and Proposition B.6 of Zhong et al., [47].

##  Appendix E Proof of Bayes Optimality of Orchestrated AMP

###  E.1 Proof of Lemma 4.3

Recall the definitions of $\bm{\Gamma}^{R}_{t,h}$ and $\bm{\Gamma}^{L}_{t,h}$
from (4.24). Using (4.9), we can write

$\displaystyle\bm{\Gamma}^{R}_{t,h}=\bm{D}^{1/2}_{h}\bm{\Sigma}^{R,\circ}_{t,h}\bm{D}^{1/2}_{h},\quad\mbox{and}\quad\bm{\Gamma}^{L}_{t,h}=\frac{1}{\gamma_{h}}\bm{D}^{1/2}_{h}\bm{\Sigma}^{L,\circ}_{t,h}\bm{D}^{1/2}_{h}.$
(E.1)

Since

$\displaystyle\mathsf{v}(\bm{\Gamma}^{R}_{t,h};\nu_{h})=\mathbb{E}[(\mathrm{V}_{h}-\mathbb{E}_{\nu_{h}}[\mathrm{V}_{h}|\mathrm{X}^{R}(\bm{\Gamma}^{R}_{t,h})])^{\otimes
2}],$ (E.2)

we have

$\mathbb{E}[\mathbb{E}_{\nu_{h}}[\mathrm{V}_{h}|\mathrm{X}^{R}(\bm{\Gamma}^{R}_{t,h})])^{\otimes
2}]]=\bm{I}_{r_{h}}-\mathsf{v}(\bm{\Gamma}^{R}_{t,h};\nu_{h}).$

Next, recall that:

$\bm{\Sigma}^{L,\circ}_{t,h}=\gamma_{h}\mathbb{E}[\mathbb{E}_{\nu_{h}}[\mathrm{V}_{h}|\bm{S}^{R,\circ}_{t,h}\mathrm{V}_{h}+(\bm{\Sigma}^{R,\circ}_{t,h})^{1/2}\mathrm{Z}^{R}_{h}]^{\otimes
2}],$

where $\mathrm{Z}^{R}_{h}\sim N_{r_{h}}(0,\bm{I}_{r_{h}})$ and is independent
of $\mathrm{V}_{h}\sim\nu_{h}$. In both the equations above and in similar
contexts later, the outer expectations are always with respect to the
distributions of both $\mathrm{V}_{h}$ and the Gaussian random variables
$\mathrm{Z}^{R}_{h}$. Again, using (4.9), the above equation can be re-
expressed as follows:

$\displaystyle\bm{\Sigma}^{L,\circ}_{t,h}$
$\displaystyle=\gamma_{h}\mathbb{E}[\mathbb{E}_{\nu_{h}}[\mathrm{V}_{h}|\bm{D}^{1/2}_{h}\mathrm{V}_{h}+\bm{D}^{1/2}_{h}(\bm{S}^{R,\circ}_{t,h})^{-1}(\bm{\Sigma}^{R,\circ}_{t,h})^{1/2}\mathrm{Z}^{R}_{h}]^{\otimes
2}]$ (E.3)
$\displaystyle=\gamma_{h}\mathbb{E}[\mathbb{E}_{\nu_{h}}[\mathrm{V}_{h}|\bm{D}^{1/2}_{h}\mathrm{V}_{h}+(\bm{\Gamma}^{R}_{t,h})^{1/2}\mathrm{Z}^{R}_{h}]^{\otimes
2}]$ (E.4)

Hence, we get

$\displaystyle\bm{\Gamma}^{L}_{t,h}$
$\displaystyle=\mathbb{E}[\mathbb{E}[\bm{D}^{1/2}_{h}\mathrm{V}_{h}|\bm{D}^{1/2}_{h}\mathrm{V}_{h}+(\bm{\Gamma}^{R}_{t,h})^{1/2}\mathrm{Z}^{R}_{h}]^{\otimes
2}]$ (E.5)
$\displaystyle=\bm{D}_{h}-\mathsf{v}_{h}(\bm{\Gamma}^{R}_{t,h};\bm{D}^{1/2}_{h}\nu_{h}).$
(E.6)

The other equation follows similarly.

###  E.2 Proof of Theorem 4.2

First, observe that
$\mathbb{E}[\mathbb{E}[\Theta|\Theta+\bm{\Gamma}^{-1/2}Z]^{\otimes
2}]=\mathbb{E}[\mathbb{E}[\Theta|\bm{\Gamma}^{1/2}\Theta+Z]^{\otimes 2}]$ for
any $\bm{Q}$ positive semi-definite and $Z\sim\mbox{Gaussian}$. Using Lemma 9
in [30], we can show that, for all $h\in[m]$:

$\displaystyle\mathsf{v}_{h}(\bm{\Gamma}^{R}_{1};\bm{D}^{1/2}_{h}\nu_{h})\succeq\mathsf{v}_{h}(\bm{\Gamma}^{R}_{2};\bm{D}^{1/2}_{h}\nu_{h})\quad\mbox{for
$\bm{\Gamma}^{R}_{1}\preceq\bm{\Gamma}^{R}_{2}$ and
$\bm{\Gamma}^{R}_{1},\bm{\Gamma}^{R}_{2}\in\mathbb{S}^{r_{h}}_{+}$.}$ (E.7)

Further, using the techniques used to prove Lemma 9 in [30], we can show that
for all $k\in[m]$ and pairs of positive semi-definite matrices
$\bm{\Gamma}^{L}_{k},\widebar{\bm{\Gamma}}^{L}_{k}\in\mathbb{S}^{r_{k}}_{+}$
satisfying $\bm{\Gamma}^{L}_{k}\preceq\widebar{\bm{\Gamma}}^{L}_{k}$, if the
matrices
$\\{\bm{\Gamma}^{L}_{g}\in\mathbb{S}^{r_{g}}_{+}:g\in[m]\setminus\\{k\\}\\}$
are fixed, then we have:

$\displaystyle\mathsf{u}_{h}(\gamma_{1}\bm{\Gamma}^{L}_{1},\ldots,\gamma_{k-1}\bm{\Gamma}^{L}_{k-1},\gamma_{k}\bm{\Gamma}^{L}_{k},\gamma_{k+1}\bm{\Gamma}^{L}_{k+1},\ldots,\gamma_{m}\bm{\Gamma}^{L}_{m};\bm{D}^{1/2}\mu)$
(E.8)
$\displaystyle\succeq\mathsf{u}_{h}(\gamma_{1}\bm{\Gamma}^{L}_{1},\ldots,\gamma_{k-1}\bm{\Gamma}^{L}_{k-1},\gamma_{k}\widebar{\bm{\Gamma}}^{L}_{k},\gamma_{k+1}\bm{\Gamma}^{L}_{k+1},\ldots,\gamma_{m}\bm{\Gamma}^{L}_{m};\bm{D}^{1/2}\mu),$
(E.9)

for all $h\in[m]$. This implies that for the collections of matrices given by
$\\{(\bm{\Gamma}^{L}_{h},\widebar{\bm{\Gamma}}^{L}_{h})\in\mathbb{S}^{r_{h}}_{+}\times\mathbb{S}^{r_{h}}_{+}:\bm{\Gamma}^{L}_{h}\preceq\widebar{\bm{\Gamma}}^{L}_{h},h\in[m]\\}$,
we have:

$\displaystyle\mathsf{u}_{h}(\gamma_{1}\bm{\Gamma}^{L}_{1},\ldots,\gamma_{m}\bm{\Gamma}^{L}_{m};\bm{D}^{1/2}\mu)\succeq\mathsf{u}_{h}(\gamma_{1}\widebar{\bm{\Gamma}}^{L}_{1},\ldots,\gamma_{m}\widebar{\bm{\Gamma}}^{L}_{m};\bm{D}^{1/2}\mu),$
(E.10)

for all $h\in[m]$. Further, by Lemma 9 in [30], we get that for any
$\\{\bm{\Gamma}^{R}_{h}\in\mathbb{S}^{r_{h}}_{+}:h\in[m]\\}$ and
$\\{\bm{\Gamma}^{L}_{h}\in\mathbb{S}^{r_{h}}_{+}:h\in[m]\\}$,

$\displaystyle\mathsf{v}_{h}(\bm{\Gamma}^{R}_{h};\bm{D}^{1/2}_{h}\nu_{h})\preceq\bm{D}_{h}\quad\mbox{and}\quad\mathsf{u}_{h}(\gamma_{1}\bm{\Gamma}^{L}_{1},\ldots,\gamma_{m}\bm{\Gamma}^{L}_{m};\bm{D}^{1/2}\mu)\preceq\bm{D}_{h}.$
(E.11)

Using Theorem 2 in [35], we can show that for all
$\bm{\Gamma}^{R}_{h}\in\mathbb{S}^{r_{h}}_{+}$ and $h\in[m]$,

$\displaystyle\mathsf{v}_{h}(\bm{\Gamma}^{R}_{h};\bm{D}^{1/2}_{h}\nu_{h})\preceq(\bm{D}^{-1}_{h}+\bm{\Gamma}^{R}_{h})^{-1}.$
(E.12)

Next, for all $h\in[m]$ and
$\\{\bm{\Gamma}^{L}_{h}\in\mathbb{S}^{r_{h}}_{+}:h\in[m]\\}$, using techniques
similar to the proof of Theorem 2 in [35], we can show the following:

$\displaystyle\mathsf{u}_{h}(\gamma_{1}\bm{\Gamma}^{L}_{1},\ldots,\gamma_{m}\bm{\Gamma}^{L}_{m};\bm{D}^{1/2}\mu)$
(E.13)
$\displaystyle\preceq(\mathsf{u}_{h}(\gamma_{1}\bm{\Gamma}^{L}_{1},\ldots,\gamma_{h-1}\bm{\Gamma}^{L}_{h-1},\bm{0},\gamma_{h+1}\bm{\Gamma}^{L}_{h+1},\ldots,\gamma_{m}\bm{\Gamma}^{L}_{m};\bm{D}^{1/2}\mu)^{-1}+\gamma_{h}\bm{\Gamma}^{L}_{h})^{-1}$
(E.14)
$\displaystyle\preceq(\bm{D}^{-1}_{h}+\gamma_{h}\bm{\Gamma}^{L}_{h})^{-1},$
(E.15)

where the last relation follows from (E.11).

Next, let us recall that
$\bm{\Sigma}^{R,\circ}_{0,h}=\bm{\Sigma}^{R,\mathrm{pc}}_{0,h}$ where
$\bm{\Sigma}^{R,\mathrm{pc}}_{0,h}$ is defined in (2.7) and construct
$\bm{\Gamma}^{R}_{0,h}$ following the prescription of (4.24). From (4.25) and
Proposition 5.6 (a) of [47], we get for all $h\in[m]$

$\displaystyle\bm{\Gamma}^{L}_{1,h}$
$\displaystyle=\bm{D}_{h}-\mathsf{v}_{h}(\bm{\Gamma}^{R}_{0,h};\bm{D}^{1/2}_{h}\nu_{h})$
(E.16)
$\displaystyle\succeq\bm{D}_{h}-(\bm{D}^{-1}_{h}+\bm{\Gamma}^{R}_{0,h})^{-1}=\mathsf{diag}\left(\frac{\gamma_{h}(\bm{D}^{4}_{h})_{ii}-1}{\gamma_{h}(\bm{D}_{h})_{ii}(1+(\bm{D}^{2}_{h})_{ii}}\right).$
(E.17)

Therefore, again using (4.25) and (E.13), we can conclude that

$\displaystyle\bm{\Gamma}^{R}_{1,h}$
$\displaystyle=\bm{D}_{h}-\mathsf{u}_{h}(\gamma_{1}\bm{\Gamma}^{L}_{1,1},\ldots,\gamma_{m}\bm{\Gamma}^{L}_{1,m};\bm{D}^{1/2}\mu)$
(E.18)
$\displaystyle\succeq\bm{D}_{h}-\left(\bm{D}^{-1}_{h}+\bm{\Gamma}^{L}_{1,h}\right)^{-1}$
(E.19)
$\displaystyle\succeq\bm{D}_{h}-\left(\bm{D}^{-1}_{h}+\mathsf{diag}\left(\frac{\gamma_{h}(\bm{D}^{4}_{h})_{ii}-1}{(\bm{D}_{h})_{ii}(1+(\bm{D}^{2}_{h})_{ii})}\right)\right)^{-1}=\bm{\Gamma}^{R}_{0,h},$
(E.20)

for all $h\in[m]$. Again, using the definitions of $\bm{\Gamma}^{L}_{1,h}$ and
$\bm{\Gamma}^{L}_{0,h}$, we can conclude using (E.7) that
$\bm{\Gamma}^{L}_{1,h}\succeq\bm{\Gamma}^{L}_{0,h}$, for all $h\in[m]$.
Consequently, using (E.7) and (E.10), we can inductively conclude that for all
$t\in\mathbb{N}$ and $h\in[m]$,
$\bm{\Gamma}^{R}_{t+1,h}\succeq\bm{\Gamma}^{R}_{t,h}$ and
$\bm{\Gamma}^{L}_{t+1,h}\succeq\bm{\Gamma}^{L}_{t,h}$. Further, using (E.7)
and (E.10), it can also be concluded that for all $t\in\mathbb{N}$ and
$h\in[m]$, $\bm{\Gamma}^{R}_{t,h}\preceq\bm{D}_{h}$ and
$\bm{\Gamma}^{L}_{t,h}\preceq\bm{D}_{h}$. Therefore, for all $h\in[m]$, there
exits matrices $\bm{\Gamma}^{R}_{h}$ and $\bm{\Gamma}^{L}_{h}$, satisfying
(4.26), such that the sequences $\\{\bm{\Gamma}^{R}_{t,h}\\}$ and
$\\{\bm{\Gamma}^{L}_{t,h}\\}$ converges to $\bm{\Gamma}^{R}_{h}$ and
$\bm{\Gamma}^{L}_{h}$, respectively, as $t\rightarrow\infty$.

###  E.3 Proof of Theorem 4.3

By simplification, we get:

$\displaystyle\frac{1}{Np_{h}}\|\bm{U}_{h}\bm{D}_{h}\bm{V}^{\top}_{h}-\widebar{\bm{U}}_{t,h}\bm{D}_{h}(\widebar{\bm{V}}_{t,h})^{\top}\|^{2}_{F}$
(E.21)
$\displaystyle=\frac{1}{Np_{h}}\mathrm{Tr}(\bm{V}^{\top}_{h}\bm{V}_{h}\bm{D}_{h}\bm{U}^{\top}_{h}\bm{U}_{h}\bm{D}_{h})+\frac{1}{Np_{h}}\mathrm{Tr}(({\bm{V}}_{h})^{\top}\widebar{\bm{V}}_{t,h}\bm{D}_{h}(\widebar{\bm{U}}_{t,h})^{\top}\widebar{\bm{U}}_{t,h}\bm{D}_{h})$
(E.22) $\displaystyle\hskip
40.00006pt-\frac{2}{Np_{h}}\mathrm{Tr}(\bm{V}^{\top}_{h}\widebar{\bm{V}}_{t,h}\bm{D}_{h}(\widebar{\bm{U}}_{t,h})^{\top}\bm{U}_{h}\bm{D}_{h}).$
(E.23)

Using the Strong Law of Large Numbers, we have:

$\frac{\bm{U}^{\top}_{h}\bm{U}_{h}}{N}\xrightarrow{a.s.}\mathbb{E}_{\mu}[U_{h}U^{\top}_{h}]\quad\mbox{and}\quad\frac{\bm{V}^{\top}_{h}\bm{V}_{h}}{p_{h}}\xrightarrow{a.s.}\mathbb{E}_{\nu_{h}}[V_{h}V^{\top}_{h}],\quad\mbox{for
all $h\in[m]$.}$

Consequently, we have:

$\frac{1}{Np_{h}}\mathrm{Tr}(\bm{V}^{\top}_{h}\bm{V}_{h}\bm{D}_{h}\bm{U}^{\top}_{h}\bm{U}_{h}\bm{D}_{h})\xrightarrow{a.s.}\mathrm{Tr}\left[\mathbb{E}_{\mu}[U_{h}U^{\top}_{h}]\bm{D}_{h}\mathbb{E}_{\nu_{h}}[V_{h}V^{\top}_{h}]\bm{D}_{h}\right].$

Using the assumption that
$\mathbb{E}_{\mu}[U_{h}U^{\top}_{h}]=\mathbb{E}_{\nu_{h}}[V_{h}V^{\top}_{h}]=\bm{I}_{r_{h}}$
for all $h\in[m]$, we have:

$\frac{1}{Np_{h}}\mathrm{Tr}(\bm{V}^{\top}_{h}\bm{V}_{h}\bm{D}_{h}\bm{U}^{\top}_{h}\bm{U}_{h}\bm{D}_{h})\xrightarrow{a.s.}\mathrm{Tr}(\bm{D}^{2}_{h}).$

Next, using Theorem 4.1 and (4.9), and the Lipschitz property of the
denoisers, we get

$\displaystyle\frac{(\widebar{\bm{U}}_{t,h})^{\top}\widebar{\bm{U}}_{t,h}}{N}\xrightarrow{a.s.}$
$\displaystyle\mathbb{E}\,[\mathbb{E}_{\mu}[U_{h}|Y^{\circ}_{t,1},\ldots,Y^{\circ}_{t,m},\widetilde{Y}_{0,1},\ldots,\widetilde{Y}_{0,\widetilde{m}}]^{\otimes
2}]=\bm{\Sigma}^{R,\circ}_{t+1,h},$ (E.24)
$\displaystyle\frac{(\widebar{\bm{V}}_{t,h})^{\top}\widebar{\bm{V}}_{t,h}}{p_{h}}\xrightarrow{a.s.}$
$\displaystyle\mathbb{E}\,[\mathbb{E}_{\nu_{h}}[V_{h}|Y^{R,\circ}_{t,h}]^{\otimes
2}]=\frac{\bm{\Sigma}^{L,\circ}_{t,h}}{\gamma_{h}}.$ (E.25)

Finally, using Theorem 4.1 and $\mathbb{E}[Yg(X)|X]=g(X)\mathbb{E}[Y|X]$, we
get

$\displaystyle\frac{(\widebar{\bm{U}}_{t,h})^{\top}\bm{U}_{h}}{N}\xrightarrow{a.s.}\bm{\Sigma}^{R,\circ}_{t+1,h},\quad\mbox{and}\quad\frac{(\widebar{\bm{V}}_{t,h})^{\top}\bm{V}_{t,h}}{p_{h}}\xrightarrow{a.s.}\frac{\bm{\Sigma}^{L,\circ}_{t,h}}{\gamma_{h}}.$
(E.26)

Plugging in (4.9) in (4.24), we get:

$\bm{\Gamma}^{R}_{t,h}:=\bm{D}^{1/2}_{h}\bm{\Sigma}^{R,\circ}_{t,h}\bm{D}^{1/2}_{h},\quad\mbox{and}\quad\bm{\Gamma}^{L}_{t,h}:=\frac{1}{\gamma_{h}}\bm{D}^{1/2}_{h}\bm{\Sigma}^{L,\circ}_{t,h}\bm{D}^{1/2}_{h}.$

Next, using Lemma 4.3, Theorem 4.2, and the fact that there exists unique
fixed points to (4.26), we get positive definite matrices
$\bm{\Gamma}^{L}_{\infty,h}$ and $\bm{\Gamma}^{R}_{\infty,h}$ such that:

$\displaystyle\lim\limits_{t\rightarrow\infty}\bm{\Sigma}^{R,\circ}_{t,h}=\bm{D}^{-1/2}_{h}\bm{\Gamma}^{R}_{\infty,h}\bm{D}^{-1/2}_{h},\quad\mbox{and}\quad\lim\limits_{t\rightarrow\infty}\bm{\Sigma}^{L,\circ}_{t+1,h}=\gamma_{h}\bm{D}^{-1/2}_{h}\bm{\Gamma}^{L}_{\infty,h}\bm{D}^{-1/2}_{h}.$
(E.27)

Substituting the above relation in (E.24) and then in (E.21), we get the
desired result.

##  Appendix F Proof of theoretical results for prediction sets

###  F.1 Proof of Theorem 4.4

Observe that as
$\widehat{\bm{R}}_{h_{k}}=\frac{1}{N}(\widebar{\bm{V}}_{T,h_{k}})_{\mathcal{F}_{h_{k}}*}\widehat{\bm{D}}_{h_{k}}$,
Theorem 4.1 implies that for all $j,\ell\in[r_{h_{k}}]\times[r_{h_{k}}]$ and
pseudo-Lipschitz functions
$\varphi_{h_{k},j,\ell}(x,v):\mathbb{R}^{2r_{h_{k}}}\rightarrow\mathbb{R}$
defined as $\varphi_{h_{k},j,\ell}(x,v)=x_{j}x_{\ell}$,

$\displaystyle\lim_{N\rightarrow\infty}N\left(\left((\widebar{\bm{V}}_{T,h_{k}})_{\mathcal{F}_{h_{k}}*}\right)^{\top}_{*j}\left((\widebar{\bm{V}}_{T,h_{k}})_{\mathcal{F}_{h_{k}}*}\right)_{*\ell}\right)$
$\displaystyle=\lim_{N\rightarrow\infty}\frac{p_{h_{k}}}{N}\times\frac{|\mathcal{F}_{h_{k}}|}{p_{h_{k}}}\times\frac{1}{|\mathcal{F}_{h_{k}}|}\sum_{i\in\mathcal{F}_{h_{k}}}\varphi_{h_{k},j,\ell}((\widebar{\bm{V}}_{T,h_{k}})_{i*},(\bm{V}_{h_{k}})_{i*})$
$\displaystyle\overset{a.s}{=}\,\lambda_{h_{k}}\gamma_{h_{k}}\mathbb{E}\left[\big{[}\mathbb{E}_{\nu_{h_{k}}}[V_{h_{k}}|Y^{R,\circ}_{T,h_{k}}]^{\otimes
2}\big{]}_{j\ell}\right],$

where $Y^{R,\circ}_{T,h_{k}}$ is defined in (4.11) and the outer expectation
is with respect to the distribution of $Y^{R,\circ}_{T,h_{k}}$. Combining
with, $\widehat{\bm{D}}_{h_{k}}\overset{a.s.}{\rightarrow}\bm{D}_{h_{k}}$, we
get

$\displaystyle\lim_{N\rightarrow\infty}N(\widehat{\bm{R}}^{\top}_{h_{k}}\widehat{\bm{R}}_{h_{k}})\,\overset{a.s}{=}\,\lambda_{h_{k}}\gamma_{h_{k}}\bm{D}_{h_{k}}\mathbb{E}\big{[}\mathbb{E}_{\nu_{h_{k}}}[V_{h_{k}}|Y^{R,\circ}_{T,h_{k}}]^{\otimes
2}\big{]}\bm{D}_{h_{k}}.$ (F.1)

Plugging in the definition of $\bm{\Sigma}^{L,\circ}_{T,h_{k}}$ from (4.9) in
(F.1), we get

$\lim_{N\rightarrow\infty}N(\widehat{\bm{R}}^{\top}_{h_{k}}\widehat{\bm{R}}_{h_{k}})\,\overset{a.s}{=}\,\lambda_{h_{k}}\bm{D}_{h_{k}}\bm{\Sigma}^{L,\circ}_{T,h_{k}}\bm{D}_{h_{k}}.$
(F.2)

Next, for any $k\in[d]$, recall that

$\frac{1}{\sqrt{N}}Q_{h_{k}}=\frac{1}{N}(\bm{V}_{h_{k}})_{\mathcal{F}_{h_{k}}*}\bm{D}_{h_{k}}U^{\mathrm{Q}}_{h_{k}}+\frac{1}{\sqrt{N}}W^{\mathrm{Q}}_{h_{k}}\,\in\,\mathbb{R}^{|\mathcal{F}_{h_{k}}|}.$

Hence, for any $k\in[d]$, we have

$\frac{1}{\sqrt{N}}(\widehat{\bm{R}}^{\top}_{h_{k}}\widehat{\bm{R}}_{h_{k}})^{-1}\widehat{\bm{R}}^{\top}_{h_{k}}Q_{h_{k}}\,=\,\widehat{U}^{\mathrm{Q},\mathsf{signal}}_{h_{k}}+\widehat{U}^{\mathrm{Q},\mathsf{noise}}_{h_{k}},$

where, conditional on
$\\{U^{\mathrm{Q}}_{h_{1}},\ldots,U^{\mathrm{Q}}_{h_{d}},\bar{\bm{V}}_{T,1},\ldots,\bar{\bm{V}}_{T,m},\widehat{\bm{D}}_{1},\ldots,\widehat{\bm{D}}_{m},{\color[rgb]{0,0,0}\bm{V}_{1},\ldots,\bm{V}_{m}}\\}$,
$\widehat{U}^{\mathrm{Q},\mathsf{signal}}_{h_{k}}$ and
$\widehat{U}^{\mathrm{Q},\mathsf{noise}}_{h_{k}}$ are independent and can be
expressed as

$\displaystyle\widehat{U}^{\mathrm{Q},\mathsf{signal}}_{h_{k}}$
$\displaystyle=\frac{1}{N}(\widehat{\bm{R}}^{\top}_{h_{k}}\widehat{\bm{R}}_{h_{k}})^{-1}\widehat{\bm{R}}^{\top}_{h_{k}}(\bm{V}_{h_{k}})^{\top}_{\mathcal{F}_{h_{k}*}}\bm{D}_{h_{k}}U^{\mathrm{Q}}_{h_{k}},$
$\displaystyle\widehat{U}^{\mathrm{Q},\mathsf{noise}}_{h_{k}}$
$\displaystyle=\frac{1}{\sqrt{N}}(\widehat{\bm{R}}^{\top}_{h_{k}}\widehat{\bm{R}}_{h_{k}})^{-1}\widehat{\bm{R}}^{\top}_{h_{k}}W^{\mathrm{Q}}_{h_{k}}.$

As
$W^{\mathrm{Q}}_{h_{k}}\overset{ind}{\sim}N_{|\mathcal{F}_{h_{k}}|}(0,\bm{I}_{|\mathcal{F}_{h_{k}}|})$
for all $k\in[d]$, we have that, conditional on
$\\{U^{\mathrm{Q}}_{h_{1}},\ldots,U^{\mathrm{Q}}_{h_{d}},\bm{V}_{1},\ldots,\bm{V}_{m},\bar{\bm{V}}_{T,1},\ldots,\newline
\bar{\bm{V}}_{T,m},\widehat{\bm{D}}_{1},\ldots,\widehat{\bm{D}}_{m}\\}$,

$\widehat{U}^{\mathrm{Q},\mathsf{noise}}_{h_{k}}\stackrel{{\scriptstyle
ind}}{{\sim}}N_{r_{h_{k}}}(0,\bm{\Sigma}^{\mathrm{Q},H}_{h_{k}}),\quad
k\in[d],$

where for each $k$,

$\bm{\Sigma}^{\mathrm{Q},H}_{h_{k}}=\frac{(\widehat{\bm{R}}^{\top}_{h_{k}}\widehat{\bm{R}}_{h_{k}})^{-1}}{N}\widehat{\bm{D}}_{h_{k}}\frac{(\widebar{\bm{V}}_{T,h_{k}})^{\top}_{\mathcal{F}_{h_{k}*}}(\widebar{\bm{V}}_{T,h_{k}})_{\mathcal{F}_{h_{k}*}}}{N}\widehat{\bm{D}}_{h_{k}}\frac{(\widehat{\bm{R}}^{\top}_{h_{k}}\widehat{\bm{R}}_{h_{k}})^{-1}}{N}.$

In view of (F.2),

$\lim_{N\to\infty}\bm{\Sigma}^{\mathrm{Q},H}_{h_{k}}=\lambda_{h_{k}}^{-1}\bm{D}_{h_{k}}^{-1}(\bm{\Sigma}^{L,\circ}_{T,h_{k}})^{-1}\bm{D}_{h_{k}}^{-1}.$
(F.3)

Using Theorem 4.1 again, we obtain that

$\displaystyle\lim_{N\rightarrow\infty}\widehat{U}^{\mathrm{Q},\mathsf{signal}}_{h_{k}}\,\overset{a.s.}{=}\,U^{\mathrm{Q}}_{h_{k}}.$
(F.4)

Combining (F.3) and (F.4), we get the stated assertion.

###  F.2 Proof of Lemma 4.4

Since $\widehat{\mu}=\widehat{\mu}_{N}\xrightarrow{w}\mu$, using arguments
similar to Lemma B.1, we obtain that the sequence of probability measures
$\\{\widehat{\mu}:N\in\mathbb{N}\\}$ is uniformly tight. Hence, for all
$\varepsilon>0$, there exists a constant $M_{\mu,\varepsilon}>0$, such that
for all $N\in\mathbb{N}$,

$\widehat{\mu}(B_{r+\widetilde{r}}(0,M_{\mu,\varepsilon}))\geq
1-\frac{\varepsilon}{8}\quad\mbox{and}\quad\mu(B_{r+\widetilde{r}}(0,M_{\mu,\varepsilon}))\geq
1-\frac{\varepsilon}{8}.$

This implies that, for all $N\in\mathbb{N}$ and
$C\in\mathscr{C}_{r+\widetilde{r}}$,

$\displaystyle\widehat{\mu}(C)-\mu(C)$
$\displaystyle=\widehat{\mu}(B_{r+\widetilde{r}}(0,M_{\mu,\varepsilon})\cap
C)-\mu(B_{r+\widetilde{r}}(0,M_{\mu,\varepsilon})\cap C)$ (F.5)
$\displaystyle~{}~{}~{}+\widehat{\mu}(B^{c}_{r+\widetilde{r}}(0,M_{\mu,\varepsilon})\cap
C)-\mu(B^{c}_{r+\widetilde{r}}(0,M_{\mu,\varepsilon})\cap C)$
$\displaystyle\leq\widehat{\mu}(B_{r+\widetilde{r}}(0,M_{\mu,\varepsilon})\cap
C)-\mu(B_{r+\widetilde{r}}(0,M_{\mu,\varepsilon})\cap
C)+\frac{\varepsilon}{4}.$

Thus, it suffices to work with the following collection of sets

$\mathscr{C}_{r+\widetilde{r},M_{\mu,\varepsilon}}=\\{C\cap
B_{r+\widetilde{r}}(0,M_{\mu,\varepsilon}):C\in\mathscr{C}_{r+\widetilde{r}}\\}.$

Note that this is a collection of compact and convex subsets of a bounded set
in $\mathbb{R}^{r+\widetilde{r}}$. Since $\mu$ is absolutely continuous with
respect to the Lebesgue measure with bounded density, Corollary 2.7.9 of [44]
implies that the collection
$\mathscr{C}_{r+\widetilde{r},M_{\mu,\varepsilon}}$ has a finite bracketing
number with respect to $\mu$. In other words, for any
$\varepsilon^{\prime}>0$, there exists a natural number,
$N_{[\,]}(\mathscr{C}_{r+\widetilde{r},M_{\mu,\varepsilon}},\varepsilon^{\prime},\mu)<\infty$
and a collection of pairs of sets,
$\mathscr{C}_{r+\widetilde{r},M_{\mu,\varepsilon},\varepsilon^{\prime},[\,]}=\left\\{(C_{L,1},C_{U,1}),\ldots,(C_{L,N_{[\,]}(\mathscr{C}_{r+\widetilde{r},M_{\mu,\varepsilon}},\varepsilon^{\prime},\mu)},C_{U,N_{[\,]}(\mathscr{C}_{r+\widetilde{r},M_{\mu,\varepsilon}},\varepsilon^{\prime},\mu)})\right\\}\subseteq\mathscr{C}_{r+\widetilde{r},M_{\mu,\varepsilon}}\times\mathscr{C}_{r+\widetilde{r},M_{\mu,\varepsilon}}$,
such that for any $C\in\mathscr{C}_{r+\widetilde{r},M_{\mu,\varepsilon}}$, we
have pairs of sets
$(C_{L,k},C_{U,k})\in\mathscr{C}_{r+\widetilde{r},M_{\mu,\varepsilon},\varepsilon^{\prime},[\,]}$
satisfying

$C_{L,k}\subseteq C\subseteq C_{U,k},\quad\mbox{and}\quad\mu(C_{U,k}\setminus
C_{L,k})<\frac{\varepsilon^{\prime}}{4}.$

Consequently, for a set in
$C\in\mathscr{C}_{r+\widetilde{r},M_{\mu,\varepsilon}}$, if its
$\varepsilon^{\prime}$-bracket is given by $(C_{L,k},C_{U,k})$, then we have

$\displaystyle\widehat{\mu}(C)-\mu(C)$
$\displaystyle\leq\widehat{\mu}(C_{U,k})-\mu(C_{U,k})+\mu(C_{U,k})-\mu(C)$
(F.6)
$\displaystyle\leq\widehat{\mu}(C_{U,k})-\mu(C_{U,k})+\frac{\varepsilon^{\prime}}{4}$
(F.7)
$\displaystyle\leq\sup\left\\{\widehat{\mu}(D_{U})-\mu(D_{U}):(D_{L},D_{U})\in\mathscr{C}_{r+\widetilde{r},M_{\mu,\varepsilon},\varepsilon^{\prime},[\,]}\right\\}+\frac{\varepsilon^{\prime}}{4}.$
(F.8)

This implies,

$\sup_{C\in\mathscr{C}_{r+\widetilde{r},M_{\mu,\varepsilon}}}\left\\{\widehat{\mu}(C)-\mu(C)\right\\}\leq\sup\left\\{\widehat{\mu}(D_{U})-\mu(D_{U}):(D_{L},D_{U})\in\mathscr{C}_{r+\widetilde{r},M_{\mu,\varepsilon},\varepsilon^{\prime},[\,]}\right\\}+\frac{\varepsilon^{\prime}}{4}.$

Similarly, we can show that

$\inf_{C\in\mathscr{C}_{r+\widetilde{r},M_{\mu,\varepsilon}}}\left\\{\widehat{\mu}(C)-\mu(C)\right\\}\geq\inf\left\\{\widehat{\mu}(D_{L})-\mu(D_{L}):(D_{L},D_{U})\in\mathscr{C}_{r+\widetilde{r},M_{\mu,\varepsilon},\varepsilon^{\prime},[\,]}\right\\}-\frac{\varepsilon^{\prime}}{4}.$

Combining the last two displays and setting
$\varepsilon^{\prime}=\varepsilon$, we obtain

$\displaystyle\sup_{C\in\mathscr{C}_{r+\widetilde{r},M_{\mu,\varepsilon}}}\left|\widehat{\mu}(C)-\mu(C)\right|$
$\displaystyle\leq\max\bigg{\\{}\sup_{(D_{L},D_{U})\in\mathscr{C}_{r+\widetilde{r},M_{\mu,\varepsilon},\varepsilon,[\,]}}\left\\{\widehat{\mu}(D_{U})-\mu(D_{U})\right\\},$
(F.9) $\displaystyle\hskip
50.00008pt-\inf_{(D_{L},D_{U})\in\mathscr{C}_{r+\widetilde{r},M_{\mu,\varepsilon},\varepsilon,[\,]}}\left\\{\widehat{\mu}(D_{L})-\mu(D_{L})\right\\}\bigg{\\}}+\frac{\varepsilon}{2}.$
(F.10)

Since, $\mu$ is absolutely continuous with respect to Lebesgue measure and
supported on $\mathbb{R}^{r+\widetilde{r}}$, it assigns zero mass to the
boundaries of compact sets. Consequently, as
$|\mathscr{C}_{r+\widetilde{r},M_{\mu,\varepsilon},\varepsilon,[\,]}|<\infty$
and $\widehat{\mu}\xrightarrow{w}\mu$, we have

$\limsup_{N\rightarrow\infty}\sup_{C\in\mathscr{C}_{r+\widetilde{r},M_{\mu,\varepsilon}}}\left|\widehat{\mu}(C)-\mu(C)\right|\leq\frac{3\varepsilon}{4},\quad\mbox{almost
surely.}$

Substituting the above inequality in (F.5), we almost surely get

$\limsup_{N\rightarrow\infty}\sup_{C\in\mathscr{C}_{r+\widetilde{r}}}\left|\widehat{\mu}(C)-\mu(C)\right|\leq\varepsilon.$

Since, $\varepsilon>0$ is arbitrary, the conclusion of the lemma follows.

###  F.3 Proof of Theorem 4.5

Let us observe that by definition, for
$y^{\mathrm{Q}}\in\mathbb{R}^{r^{\mathrm{Q}}}$ and
$\widetilde{y}^{\mathrm{Q}}\in\mathbb{R}^{\widetilde{r}^{\mathrm{Q}}}$
satisfying (2.33), we have

$\displaystyle\mathbb{P}_{\widehat{\mu}}\left[(U^{\mathrm{Q}}_{1},\ldots,U^{\mathrm{Q}}_{m},\widetilde{U}^{\mathrm{Q}}_{1},\ldots,\widetilde{U}^{\mathrm{Q}}_{\widetilde{m}})\in\mathcal{C}_{\alpha}\,\Big{|}\,Y^{\mathrm{Q}}=y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}}=\widetilde{y}^{\mathrm{Q}}\right]\geq
1-\alpha,$ (F.11)

where $Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}}$ are defined in (2.31) with
prior $\widehat{\mu}$. Let us define the function:

$f(x;B,\pi):=\mathbb{P}_{\pi}\left[(U^{\mathrm{Q}}_{1},\ldots,U^{\mathrm{Q}}_{m},\widetilde{U}^{\mathrm{Q}}_{1},\ldots,\widetilde{U}^{\mathrm{Q}}_{\widetilde{m}})\in
B\,\Big{|}\,(Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})=x\right].$

In the above definition,
$x\in\mathbb{R}^{r^{\mathrm{Q}}+\widetilde{r}^{\mathrm{Q}}}$,
$B\subset\mathbb{R}^{r+\widetilde{r}}$ and
$(Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})$ satisfy (2.31) with

$(U^{\mathrm{Q}}_{h_{1}},\ldots,U^{\mathrm{Q}}_{h_{d}},\widetilde{U}^{\mathrm{Q}}_{\ell_{1}},\ldots,\widetilde{U}^{\mathrm{Q}}_{\ell_{\widetilde{d}}})\sim\pi_{h_{1},\ldots,h_{d};\ell_{1},\ldots,\ell_{\widetilde{d}}}.$

Here, $\pi_{h_{1},\ldots,h_{d};\ell_{1},\ldots,\ell_{\widetilde{d}}}$ is the
marginal distribution of the modalities $h_{1},\ldots,h_{d}$ and
$\ell_{1},\ldots,\ell_{\widetilde{d}}$ when the entire set of latent factors
$(U^{\mathrm{Q}}_{1},\ldots,U^{\mathrm{Q}}_{m},\widetilde{U}^{\mathrm{Q}}_{1},\ldots,\widetilde{U}^{\mathrm{Q}}_{\widetilde{m}})\sim\pi$.
Now, Theorem 4.4, and Skorokhod’s representation theorem implies that
$y^{\mathrm{Q}}=s^{\mathrm{Q}}+e^{\mathrm{Q}}$, where $s^{\mathrm{Q}}$ is a
sample from the distribution of $Y^{\mathrm{Q}}$ and $e^{\mathrm{Q}}$ is a
realization of a random variable
$E^{\mathrm{Q}}=(E^{\mathrm{Q}}_{h_{1}},\ldots,E^{\mathrm{Q}}_{h_{d}})\in\mathbb{R}^{r^{\mathrm{Q}}}$
defined as follows:

$E^{\mathrm{Q}}_{h_{k}}=\left(\frac{1}{N}(\widehat{\bm{R}}^{\top}_{h_{k}}\widehat{\bm{R}}_{h_{k}})^{-1}\widehat{\bm{R}}^{\top}_{h_{k}}(\bm{V}_{h_{k}})^{\top}_{\mathcal{F}_{h_{k}*}}\bm{D}_{h_{k}}-\bm{I}_{r_{h_{k}}}\right)U^{\mathrm{Q}}_{h_{k}}+\bm{T}_{h_{k}}Z_{h_{k}}.$

In the above definition, the random matrix $\bm{T}_{h_{k}}\xrightarrow{a.s}0$
and $Z_{h_{k}}\sim N_{r_{h_{k}}}(0,\bm{I}_{r_{h_{k}}})$. Conditional on

$\\{\bar{\bm{V}}_{T,1},\ldots,\bar{\bm{V}}_{T,m},\widehat{\bm{D}}_{1},\ldots,\widehat{\bm{D}}_{m},\bm{V}_{1},\ldots,\bm{V}_{m}\\},$
$\displaystyle E^{\mathrm{Q}}_{h_{k}}\sim
N\left(\left(\frac{1}{N}(\widehat{\bm{R}}^{\top}_{h_{k}}\widehat{\bm{R}}_{h_{k}})^{-1}\widehat{\bm{R}}^{\top}_{h_{k}}(\bm{V}_{h_{k}})^{\top}_{\mathcal{F}_{h_{k}*}}\bm{D}_{h_{k}}-\bm{I}_{r_{h_{k}}}\right)U^{\mathrm{Q}}_{h_{k}},\bm{T}_{h_{k}}{\color[rgb]{0,0,0}{\bm{T}}_{h_{k}}^{\top}}\right),$
(F.12)

and all quantities defining mean and variance in the last display, except for
$U^{\mathrm{Q}}_{h_{k}}$, are deterministic after conditioning. However, from
Lemma 4.1 and Theorem 4.1, we can conclude that

$\frac{1}{N}(\widehat{\bm{R}}^{\top}_{h_{k}}\widehat{\bm{R}}_{h_{k}})^{-1}\widehat{\bm{R}}^{\top}_{h_{k}}(\bm{V}_{h_{k}})^{\top}_{\mathcal{F}_{h_{k}*}}\bm{D}_{h_{k}}-\bm{I}_{r_{h_{k}}}\xrightarrow{a.s}0.$

Henceforth, we shall condition on the the reference data
$\\{\bar{\bm{V}}_{T,1},\ldots,\bar{\bm{V}}_{T,m},\widehat{\bm{D}}_{1},\ldots,\widehat{\bm{D}}_{m},\bm{V}_{1},\ldots,\bm{V}_{m}\\}$.
Since, almost surely $\widehat{\mu}\stackrel{{\scriptstyle w}}{{\to}}\mu$, for
a given $\varepsilon>0$, we can get a constant $K>0$ (independent of $N$) such
that

$\displaystyle\mathbb{P}_{\widehat{\mu}}[\|(U^{\mathrm{Q}}_{h_{1}},\ldots,U^{\mathrm{Q}}_{h_{d}})\|>K]<\frac{\varepsilon}{16}\quad\mbox{a.s.}\quad\mbox{and}\quad\mathbb{P}_{\mu}[\|(U^{\mathrm{Q}}_{h_{1}},\ldots,U^{\mathrm{Q}}_{h_{d}})\|>K]<\frac{\varepsilon}{16}.$
(F.13)

Therefore, by (F.12), (F.13) and the same $\varepsilon>0$ chosen in (F.13), we
can get a constant $\widebar{M}>0$ (independent of $N$) such that

$\displaystyle\mathbb{P}_{\widehat{\mu}}[\|E^{\mathrm{Q}}\|>\widebar{M}]<\frac{\varepsilon}{16}\quad\mbox{a.s.}\quad\mbox{and}\quad\mathbb{P}_{\mu}[\|E^{\mathrm{Q}}\|>\widebar{M}]<\frac{\varepsilon}{16}.$
(F.14)

Henceforth, we shall use
$\mathbb{P}_{\widehat{\mu}}\left[(U^{\mathrm{Q}}_{1},\ldots,U^{\mathrm{Q}}_{m},\widetilde{U}^{\mathrm{Q}}_{1},\ldots,\widetilde{U}^{\mathrm{Q}}_{\widetilde{m}})\in\mathcal{C}_{\alpha}\,\Big{|}\,Y^{\mathrm{Q}}+E^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}}\right]$
as a (slightly abused) alternative notation for
$f((Y^{\mathrm{Q}}+E^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}});\mathcal{C}_{\alpha},\widehat{\mu})$
and
$\mathbb{P}_{\mu}\left[(U^{\mathrm{Q}}_{1},\ldots,U^{\mathrm{Q}}_{m},\widetilde{U}^{\mathrm{Q}}_{1},\ldots,\widetilde{U}^{\mathrm{Q}}_{\widetilde{m}})\in\mathcal{C}_{\alpha}\,\Big{|}\,Y^{\mathrm{Q}}+E^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}}\right]$
for
$f((Y^{\mathrm{Q}}+E^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}});\mathcal{C}_{\alpha},\mu)$.
From (F.11), we have

$\mathbb{E}_{\widehat{\mu}}\left[\mathbb{P}_{\widehat{\mu}}\left[(U^{\mathrm{Q}}_{1},\ldots,U^{\mathrm{Q}}_{m},\widetilde{U}^{\mathrm{Q}}_{1},\ldots,\widetilde{U}^{\mathrm{Q}}_{\widetilde{m}})\in\mathcal{C}_{\alpha}\,\Big{|}\,Y^{\mathrm{Q}}+E^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}}\right]\right]\geq
1-\alpha.$

We shall show that, as $N\rightarrow\infty$,

$\displaystyle\bigg{|}\mathbb{E}_{\widehat{\mu}}\left[\mathbb{P}_{\widehat{\mu}}\left[(U^{\mathrm{Q}}_{1},\ldots,U^{\mathrm{Q}}_{m},\widetilde{U}^{\mathrm{Q}}_{1},\ldots,\widetilde{U}^{\mathrm{Q}}_{\widetilde{m}})\in\mathcal{C}_{\alpha}\,\Big{|}\,Y^{\mathrm{Q}}+E^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}}\right]\right]$
(F.15) $\displaystyle\hskip
100.00015pt-\mathbb{E}_{\widehat{\mu}}\left[\mathbb{P}_{\widehat{\mu}}\left[(U^{\mathrm{Q}}_{1},\ldots,U^{\mathrm{Q}}_{m},\widetilde{U}^{\mathrm{Q}}_{1},\ldots,\widetilde{U}^{\mathrm{Q}}_{\widetilde{m}})\in\mathcal{C}_{\alpha}\,\Big{|}\,Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}}\right]\right]\bigg{|}\xrightarrow{a.s}0.$

This implies

$\limsup_{n\rightarrow\infty}\mathbb{P}_{\widehat{\mu}}\left[(U^{\mathrm{Q}}_{1},\ldots,U^{\mathrm{Q}}_{m},\widetilde{U}^{\mathrm{Q}}_{1},\ldots,\widetilde{U}^{\mathrm{Q}}_{\widetilde{m}})\in\mathcal{C}_{\alpha}\right]\geq
1-\alpha,\quad\mbox{a.s.}$

Now since $\widehat{\mu}\xrightarrow{w}\mu$, the sequences of random variables
$\\{(Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})\\}$ and
$\\{(Y^{\mathrm{Q}}+E^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})\\}$ are tight.
This implies, for the $\varepsilon>0$ chosen in (F.13), there exist compact
and convex sets $M\in\mathbb{R}^{r^{\mathrm{Q}}+\widetilde{r}^{\mathrm{Q}}}$
and $\widetilde{M}\in\mathbb{R}^{r+\widetilde{r}}$ such that

$\displaystyle\mathbb{P}_{\widehat{\mu}}[(Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})\notin
M]<\frac{\varepsilon}{16},\quad\mathbb{P}_{\widehat{\mu}}[(Y^{\mathrm{Q}}+E^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})\notin
M]<\frac{\varepsilon}{16},$ (F.16)
$\displaystyle\mathbb{P}_{\mu}[(Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})\notin
M]<\frac{\varepsilon}{16},\quad\mathbb{P}_{\mu}[(Y^{\mathrm{Q}}+E^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})\notin
M]<\frac{\varepsilon}{16},$
$\displaystyle\widehat{\mu}(\widetilde{M}^{c})<\frac{\varepsilon}{16},\quad\mbox{and}\quad\mu(\widetilde{M}^{c})<\frac{\varepsilon}{16}.$

In the above definition,
$\mathbb{P}_{\mu}[(Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})\in\cdot]$ refers
to the probability of $(Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})$ when in
the definition (2.31), the latent factors
$(U^{\mathrm{Q}}_{1},\ldots,U^{\mathrm{Q}}_{m},\widetilde{U}^{\mathrm{Q}}_{1},\ldots,\widetilde{U}^{\mathrm{Q}}_{\widetilde{m}})\sim\mu$.
By properties of multivariate Gaussian distributions, for any compact set
$M\in\mathbb{R}^{r^{\mathrm{Q}}+\widetilde{r}^{\mathrm{Q}}}$ and
$t\in\mathbb{R}^{r^{\mathrm{Q}}}$ satisfying $\|t\|\leq\widebar{M}$ (with
$\widebar{M}$ defined in (F.14)), the function

$\displaystyle
f((y+t,\widetilde{y});B,\pi):=\mathbb{P}_{\pi}\left[(U^{\mathrm{Q}}_{1},\ldots,U^{\mathrm{Q}}_{m},\widetilde{U}^{\mathrm{Q}}_{1},\ldots,\widetilde{U}^{\mathrm{Q}}_{\widetilde{m}})\in
B\,\Big{|}\,(Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})=(y+t,\widetilde{y})\right]\mathbbm{1}_{[(y,\widetilde{y})\in
M]},$

is uniformly Lipschitz and bounded as a function of $(y,\widetilde{y})\in M$.
By Corollary 2.7.2 of [44], the class of functions

$\mathcal{F}_{M}=\bigg{\\{}f((y+t,{\widetilde{y}});B,\pi)\mathbbm{1}_{[(y,\widetilde{y})\in
M]}:\pi\in\\{\widehat{\mu},\mu\\},\,B\subset\widetilde{M},\,\|t\|\leq\widebar{M}\bigg{\\}},$

has a finite bracketing number. Therefore, proceeding as in the proof of Lemma
4.4, we can show that

$\sup_{\begin{subarray}{c}\pi\in\\{\widehat{\mu},\mu\\}\\\
B\subset\widetilde{M},\|t\|\leq\widebar{M}\end{subarray}}\bigg{|}\mathbb{E}_{\widehat{\mu}}\left[f((Y^{\mathrm{Q}}+t,\widetilde{Y}^{\mathrm{Q}}),B,\pi)\mathbbm{1}_{[(Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})\in
M]}\right]-\mathbb{E}_{\mu}\left[f((Y^{\mathrm{Q}}+t,\widetilde{Y}^{\mathrm{Q}}),B,\pi)\mathbbm{1}_{[(Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})\in
M]}\right]\bigg{|}\xrightarrow{a.s}0.$ (F.17)

This implies

$\displaystyle\bigg{|}\mathbb{E}_{\widehat{\mu}}\left[f((Y^{\mathrm{Q}}+E^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}}),\mathcal{C}_{\alpha},\widehat{\mu})\mathbbm{1}_{[(Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})\in
M,\,\|E^{\mathrm{Q}}\|\leq\widebar{M}]}\right]$ (F.18) $\displaystyle\hskip
60.00009pt-\mathbb{E}_{\mu}\left[f((Y^{\mathrm{Q}}+E^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}}),\mathcal{C}_{\alpha},\widehat{\mu})\mathbbm{1}_{[(Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})\in
M,\,\|E^{\mathrm{Q}}\|\leq\widebar{M}]}\right]\bigg{|}\xrightarrow{a.s}0.$

Similarly, we can show that

$\displaystyle\bigg{|}\mathbb{E}_{\widehat{\mu}}\left[f((Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}}),\mathcal{C}_{\alpha},\widehat{\mu})\mathbbm{1}_{[(Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})\in
M]}\right]-\mathbb{E}_{\mu}\left[f((Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}}),\mathcal{C}_{\alpha},\widehat{\mu})\mathbbm{1}_{[(Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})\in
M]}\right]\bigg{|}\xrightarrow{a.s}0.$ (F.19)

Now, let us consider the following:

$\displaystyle\bigg{|}\mathbb{E}_{\widehat{\mu}}\left[\mathbb{P}_{\widehat{\mu}}\left[(U^{\mathrm{Q}}_{1},\ldots,U^{\mathrm{Q}}_{m},\widetilde{U}^{\mathrm{Q}}_{1},\ldots,\widetilde{U}^{\mathrm{Q}}_{\widetilde{m}})\in\mathcal{C}_{\alpha}\,\Big{|}\,Y^{\mathrm{Q}}+E^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}}\right]\right]$
(F.20) $\displaystyle\hskip
100.00015pt-\mathbb{E}_{\widehat{\mu}}\left[\mathbb{P}_{\widehat{\mu}}\left[(U^{\mathrm{Q}}_{1},\ldots,U^{\mathrm{Q}}_{m},\widetilde{U}^{\mathrm{Q}}_{1},\ldots,\widetilde{U}^{\mathrm{Q}}_{\widetilde{m}})\in\mathcal{C}_{\alpha}\,\Big{|}\,Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}}\right]\right]\bigg{|}$
$\displaystyle\hskip
20.00003pt=\bigg{|}\mathbb{E}_{\widehat{\mu}}\left[\mathbb{P}_{\widehat{\mu}}\left[(U^{\mathrm{Q}}_{1},\ldots,U^{\mathrm{Q}}_{m},\widetilde{U}^{\mathrm{Q}}_{1},\ldots,\widetilde{U}^{\mathrm{Q}}_{\widetilde{m}})\in\mathcal{C}_{\alpha}\,\Big{|}\,Y^{\mathrm{Q}}+E^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}}\right]\mathbbm{1}_{[(Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})\notin
M]}\right]$ $\displaystyle\hskip
50.00008pt+\mathbb{E}_{\widehat{\mu}}\left[\mathbb{P}_{\widehat{\mu}}\left[(U^{\mathrm{Q}}_{1},\ldots,U^{\mathrm{Q}}_{m},\widetilde{U}^{\mathrm{Q}}_{1},\ldots,\widetilde{U}^{\mathrm{Q}}_{\widetilde{m}})\in\mathcal{C}_{\alpha}\,\Big{|}\,Y^{\mathrm{Q}}+E^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}}\right]\mathbbm{1}_{[(Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})\in
M]}\mathbbm{1}_{[\|E^{\mathrm{Q}}\|>\widebar{M}]}\right]$ $\displaystyle\hskip
70.0001pt+\mathbb{E}_{\widehat{\mu}}\left[\mathbb{P}_{\widehat{\mu}}\left[(U^{\mathrm{Q}}_{1},\ldots,U^{\mathrm{Q}}_{m},\widetilde{U}^{\mathrm{Q}}_{1},\ldots,\widetilde{U}^{\mathrm{Q}}_{\widetilde{m}})\in\mathcal{C}_{\alpha}\,\Big{|}\,Y^{\mathrm{Q}}+E^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}}\right]\mathbbm{1}_{[(Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})\in
M,\|E^{\mathrm{Q}}\|\leq\widebar{M}]}\right]$ $\displaystyle\hskip
100.00015pt-\mathbb{E}_{\widehat{\mu}}\left[\mathbb{P}_{\widehat{\mu}}\left[(U^{\mathrm{Q}}_{1},\ldots,U^{\mathrm{Q}}_{m},\widetilde{U}^{\mathrm{Q}}_{1},\ldots,\widetilde{U}^{\mathrm{Q}}_{\widetilde{m}})\in\mathcal{C}_{\alpha}\,\Big{|}\,Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}}\right]\mathbbm{1}_{[(Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})\in
M]}\right]$ $\displaystyle\hskip
130.0002pt-\mathbb{E}_{\widehat{\mu}}\left[\mathbb{P}_{\widehat{\mu}}\left[(U^{\mathrm{Q}}_{1},\ldots,U^{\mathrm{Q}}_{m},\widetilde{U}^{\mathrm{Q}}_{1},\ldots,\widetilde{U}^{\mathrm{Q}}_{\widetilde{m}})\in\mathcal{C}_{\alpha}\,\Big{|}\,Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}}\right]\mathbbm{1}_{[(Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})\notin
M]}\right]\bigg{|}$ $\displaystyle\hskip 20.00003pt\leq
2\,\mathbb{P}_{\widehat{\mu}}[(Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})\notin
M]+\mathbb{P}_{\widehat{\mu}}[\|E^{\mathrm{Q}}\|>\widebar{M}]+T_{1}+T_{2}+T_{3}$
$\displaystyle\hskip 20.00003pt\leq\frac{\varepsilon}{4}+T_{1}+T_{2}+T_{3},$

where

$\displaystyle T_{1}$
$\displaystyle=\bigg{|}\mathbb{E}_{\widehat{\mu}}\left[f((Y^{\mathrm{Q}}+E^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}}),\mathcal{C}_{\alpha},\widehat{\mu})\mathbbm{1}_{[(Y^{\mathrm{Q}},\widetilde{Y}^{\mathrm{Q}})\in
M,\,\|E^{\mathrm{Q}}\|\leq\widebar{M}]}\right]$ $\displaystyle\hskip