\begin{equation} \label{eq: gradient-connect-general1}
\begin{split}
\grad f(\X) &= \overline{\grad_{\L}\, h_{r}([\L,\R])} \W_{\L,\R} \R^\dagger  + ( \overline{\grad_{\R}\, h_{r}([\L,\R])} \V_{\L,\R} \L^\dagger )^\top (\I_{p_2} - \R \R^\dagger) \\
\overline{\grad\, h_{r}([\L,\R])} &= \begin{bmatrix}
\grad f(\X) \R \W_{\L,\R}^{-1} \\
(\grad f(\X))^\top \L \V_{\L,\R}^{-1}
\end{bmatrix}.
\end{split}
\end{equation}

Furthermore, if $[\L,\R]$ is a Riemannian FOSP of (<ref>), we have:

\begin{equation} \label{eq: Hessian-connection-general1}
\begin{split}
\overline{\Hess \, h_{r}([\L,\R])}[\theta_{(\L,\R)}, \theta_{(\L,\R)}]  &= \Hess f(\X)[\cL_{\L,\R}^{r}(\theta_{(\L,\R)}),\cL_{\L,\R}^{r}(\theta_{(\L,\R)})], \quad \forall \theta_{(\L,\R)} \in \cH_{(\L,\R)} \widebar{\cM}_{r}^{q_1}.
\end{split}
\end{equation}

Finally, $\overline{\Hess \, h_{r}([\L,\R])}$ and $\Hess f(\X)$ have $(p_1 + p_2-r)r$ eigenvalues and for $i = 1,\ldots, (p_1 + p_2-r)r$, we have
$\lambda_i(\overline{\Hess \, h_{r}([\L,\R])})$ is sandwiched between $\overline{\gamma} \lambda_i(\Hess f(\X)) $ and $2\underline{\Gamma} \lambda_i(\Hess f(\X)) $, where $\overline{\gamma} = \sup_{\L, \R: \L \R^\top = \X} \gamma_{\L,\R}$ and $\underline{\Gamma}  = \inf_{\L, \R: \L \R^\top = \X} \Gamma_{\L,\R}$ where $\gamma_{\L,\R}$ and $\Gamma_{\L,\R}$ are given in (<ref>).

Proof of Theorem <ref>. First, recall $\U$ and $\V$ span the top $r$ left and right singular subspaces of $\L\R^\top$, respectively and we have $\L \L^\dagger = P_\U$, $\R \R^\dagger = P_\V$. So (<ref>) is by direct calculation from the gradient expressions in Proposition <ref>.

Next, we prove (<ref>). Since $[\L,\R]$ is a Riemannian FOSP of (<ref>), we have

\begin{equation} \label{eq: FOSP-condition-general1}
\overline{ \grad \, h_{r}([\L,\R])} = \0,\quad \nabla f(\L \R^\top) \R = \0 \quad \text{ and }  \left(\nabla f(\L \R^\top) \right)^\top \L = \0
\end{equation}
and have $\grad f(\X) = \0$ by (<ref>). So $ \nabla f(\X) = P_{\U_\perp} \nabla f(\X) P_{\V_\perp}$. For any $\L, \R$ such that $\L \R^\top = \X$. Recall $\P_1 = \U^\top \L$, $\P_2 = \V^\top \R$, and let $\bSigma = \U^\top \X \V$. Given any $\theta_{(\L,\R)} =  [\theta_L^\top \quad \theta_R^\top]^\top \in \cH_{(\L,\R)} \widebar{\cM}_{r}^{q_1}$, we have

\begin{equation} \label{eq: Hessian-con-gradient-general1}
\langle \nabla f(\X), P_{\U_\perp} \theta_L \P_2^{\top} \bSigma^{-1} \P_1 \theta_R^\top  P_{\V_\perp}    \rangle = \langle \nabla f(\X), \theta_L \theta_R^\top \rangle,
\end{equation}
where the equality is because $\P_1,\P_2$ are  nonsingular, $\P_1 \P_2^\top = \bSigma$ and $ \nabla f(\X) = P_{\U_\perp} \nabla f(\X) P_{\V_\perp} $.

Then by Proposition <ref>:

\begin{equation*}
\begin{split}
&\quad \overline{\Hess \, h_{r}([\L,\R])}[\theta_{(\L,\R)}, \theta_{(\L,\R)}] \\
&=  \nabla^2 f(\L \R^\top)[\L \theta_R^\top + \theta_L \R^\top,\L \theta_R^\top + \theta_L \R^\top] + 2 \langle \nabla f(\L \R^\top), \theta_L \theta_R^\top \rangle \\
& \quad + \langle \nabla f(\L \R^\top) \R \rmD \W_{\L,\R}^{-1}[\theta_{(\L,\R)}] , \theta_L \W_{\L,\R} \rangle + \langle  \left(\nabla f(\L \R^\top) \right)^\top \L \rmD \V_{\L,\R}^{-1}[\theta_{(\L,\R)}], \theta_R \V_{\L,\R} \rangle \\
& \quad +  \langle \rmD \W_{\L,\R}[ \overline{ \grad \, h_{r}([\L,\R])} ], \theta_L^\top \theta_L \rangle /2 +  \langle \rmD \V_{\L,\R}[ \overline{ \grad \, h_{r}([\L,\R])} ], \theta_R^\top \theta_R \rangle /2\\
& \overset{ \eqref{eq: FOSP-condition-general1} } =  \nabla^2 f(\L \R^\top)[\L \theta_R^\top + \theta_L \R^\top,\L \theta_R^\top + \theta_L \R^\top] + 2 \langle \nabla f(\L \R^\top), \theta_L \theta_R^\top \rangle \\
& \overset{ \text{Proposition } \ref{prop: general-bijection1}, \eqref{eq: Hessian-con-gradient-general1} } =   \nabla^2 f(\X)[\cL^r_{\L,\R}(\theta_{(\L,\R)}),\cL^r_{\L,\R}(\theta_{(\L,\R)})] + 2 \langle \nabla f(\X), P_{\U_\perp} \theta_L \P_2^{\top} \bSigma^{-1} \P_1 \theta_R^\top  P_{\V_\perp}    \rangle\\
& = \Hess f(\X)[\cL^r_{\L,\R}(\theta_{(\L,\R)}),\cL^r_{\L,\R}(\theta_{(\L,\R)})],
\end{split}
\end{equation*}
where the last equality follows from the expression of $\Hess f(\X)$ in (<ref>) and the definition of $\cL^r_{\L,\R}$.

Then, by (<ref>), (<ref>) and Theorem <ref>, we have $\overline{\Hess \, h_{r}([\L,\R])}$ and $\Hess f(\X)$ have $(p_1+p_2- r)r$ eigenvalues and $\widebar{\lambda}_i(\Hess \, h_{r}([\L,\R]))$ is sandwiched between $\gamma_{\L,\R} \lambda_i(\Hess f(\X)) $ and $2\Gamma_{\L,\R} \lambda_i(\Hess f(\X)) $ for $i = 1,\ldots,(p_1+p_2- r)r$, where $\gamma_{\L,\R}$ and $\Gamma_{\L,\R}$ are given in (<ref>). Since the same sandwich relationship holds for all $(\L, \R)$ such that $\L \R^\top = \X$, we have
$\lambda_i(\overline{\Hess \, h_{r}([\L,\R])})$ is sandwiched between $\overline{\gamma} \lambda_i(\Hess f(\X)) $ and $2\underline{\Gamma} \lambda_i(\Hess f(\X)) $.   $\blacksquare$

Suppose $\U \in \st(r,p_1)$, $\B \in \bbS_+(r)$, $\V \in \st(p_2,r)$ and $\X = \U \B \V^\top$. Then

\begin{equation} \label{eq: gradient-connect-general2}
\begin{split}
\grad f(\X) &= P_{\U_\perp} \overline{\grad_\U\, h_{r}([\U,\Y])} \B^{-1} \V^\top + \U \bDelta_1 \V^\top + (P_{\V_\perp} \overline{\grad_\V\, h_{r}([\U,\B,\V])} \B^{-1} \U^\top)^\top
\end{split}
\end{equation}
where $\bDelta_1$ is uniquely determined by the equation system:

\begin{equation*}
\begin{split}
&\B\skew^\top(\bDelta_1 ) + \skew^\top(\bDelta_1 ) \B = 2 \overline{\grad_\U\, h_{r}([\U,\Y])}^\top \U,\\
&\sym(\bDelta_1) = \B^{-1} \overline{\grad_\B\, h_{r}([\U,\Y])} \B^{-1},
\end{split}
\end{equation*}

\begin{equation}  \label{eq: gradient-connect-general2-secondone}
\begin{split}
\overline{\grad\, h_{r}([\U,\B,\V])} &= \begin{bmatrix}
P_{\U_\perp}\grad f(\X) \V \B + \U ( \skew(\bDelta_2) \B + \B \skew(\bDelta_2)  )/2 \\
\B \sym(\bDelta_2) \\
(\B \U^\top \grad f(\X) P_{\V_\perp} )^\top - \V ( \skew(\bDelta_2) \B + \B \skew(\bDelta_2)  )/2
\end{bmatrix},
\end{split}
\end{equation}
where $\bDelta_2 = \U^\top \grad f(\X) \V$.

Furthermore, if $[\U,\B,\V]$ is a Riemannian FOSP of (<ref>), we have: for any $ \theta_{(\U,\B,\V)} \in \cH_{(\U,\B,\V)} \widebar{\cM}_{r}^{q_2}$,

\begin{equation} \label{eq: Hessian-connection-general2}
\begin{split}
\overline{\Hess \, h_{r}([\U,\B,\V])}[\theta_{(\U,\B,\V)}, \theta_{(\U,\B,\V)}]  &= \Hess f(\X)[\cL_{\U,\B,\V}^{r}(\theta_{(\U,\B,\V)}),\cL_{\U,\B,\V}^{r}(\theta_{(\U,\B,\V)})].
\end{split}
\end{equation}

Finally, $\overline{\Hess \, h_{r}([\U,\B,\V])}$ has $(p_1 + p_2-r)r$ eigenvalues and for $i = 1,\ldots, (p_1 + p_2-r)r$, we have
$\lambda_i(\overline{\Hess \, h_{r}([\U,\B,\V])})$ is sandwiched between $\sigma^2_r(\X)\lambda_i(\Hess f(\X)) $ and $2\sigma^2_1(\X) \lambda_i(\Hess f(\X)) $.

Proof of Theorem <ref>. First, recall $\bDelta = \U^\top \nabla f(\U \B \V^\top) \V$ and notice $\B\skew^\top(\bDelta_1 ) + \skew^\top(\bDelta_1 ) \B = 2 \overline{\grad_\U\, h_{r}([\U,\Y])}^\top \U = \B\skew^\top(\bDelta ) + \skew^\top(\bDelta ) \B$ is a Sylvester equation with respect $\skew(\bDelta_1)^\top$, which has a unique solution as $\B,-\B$ have disjoint spectra and <cit.>. Since $\bDelta_1 = \skew(\bDelta_1) + \sym(\bDelta_1)$, $\bDelta_1$ is uniquely determined by the equation system $ \B\skew^\top(\bDelta_1 ) + \skew^\top(\bDelta_1 ) \B = 2 \overline{\grad_\U\, h_{r}([\U,\Y])}^\top \U, \sym(\bDelta_1) = \B^{-1} \overline{\grad_\B\, h_{r}([\U,\Y])} \B^{-1}$. Finally, because $\bDelta$ is a solution to this equation system, we have $\bDelta_1 = \bDelta = \U^\top \nabla f(\U \B \V^\top) \V$. The rest proofs of (<ref>) and (<ref>) are by direct calculation from the gradient expressions in Proposition <ref>.

Next, we prove (<ref>). Since $[\U,\B,\V]$ is a Riemannian FOSP of (<ref>), we have

\begin{equation} \label{eq: FOSP-condition-general2}
\overline{ \grad \, h_{r}([\U,\B,\V])} = \0,\quad \grad f(\X) \overset{\eqref{eq: gradient-connect-general2}}= \0, \quad \nabla f(\U \B \V^\top) \V = \0 \,\text{ and }\,   \U^\top\nabla f(\U \B \V^\top)  = \0.
\end{equation}
So $ \nabla f(\X) = P_{\U_\perp} \nabla f(\X) P_{\V_\perp}$. Given any $\theta_{(\U,\B,\V)} =  [\theta_U^\top \quad \theta_B^\top \quad \theta_V^\top]^\top \in \cH_{(\U,\B,\V)} \widebar{\cM}_{r}^{q_2}$, we have

\begin{equation} \label{eq: Hessian-con-gradient-general2}
\langle \nabla f(\X), P_{\U_\perp} \theta_U \B \B^{-1} \B \theta_V^\top  P_{\V_\perp}    \rangle = \langle \nabla f(\X), \theta_U \B \theta_V^\top \rangle,
\end{equation}
where the equality is because $ \nabla f(\X) = P_{\U_\perp} \nabla f(\X) P_{\V_\perp} $.

Then by Proposition <ref> and recall $\bDelta' =  \theta_U^\top\nabla f(\U \B \V^\top) \V$ and $\bDelta'' =  \U^\top\nabla f(\U \B \V^\top) \theta_V$, we have

\begin{equation*}
\begin{split}
&\quad \overline{\Hess \, h_{r}([\U,\B,\V])}[\theta_{(\U,\B,\V)}, \theta_{(\U,\B,\V)}] \\
&=  \nabla^2 f(\U \B \V^\top)[\theta_U \B \V^\top + \U \theta_B \V^\top + \U \B \theta_V^\top, \theta_U \B \V^\top + \U \theta_B \V^\top + \U \B \theta_V^\top] + 2\langle \nabla f(\U\B\V^\top), \theta_U \B \theta_V^\top \rangle\\
& +   \left\langle \bDelta,  \sym(\U^\top \theta_U \U^\top\theta_U) \B + \B \sym(\V^\top \theta_V \U^\top \theta_U) -2\theta_U^\top \theta_U \B \right\rangle/2\\
& + \left\langle \bDelta,  \B\sym(\V^\top \theta_V \V^\top\theta_V) + \sym(\U^\top \theta_U \V^\top \theta_V) \B -2\B\theta_V^\top \theta_V +  2\theta_B \B^{-1} \theta_B \right\rangle/2\\
& + \langle \bDelta', 2\theta_B -  \U^\top \theta_U \B - \theta_U^\top \U \B/2 -\V^\top \theta_V \B/2 \rangle + \langle \bDelta'', 2\theta_B -  \B\theta_V^\top \V - \B\V^\top \theta_V/2 -\B \theta_U^\top \U /2 \rangle,\\
& \overset{ \eqref{eq: FOSP-condition-general2} } =  \nabla^2 f(\U \B \V^\top)[\theta_U \B \V^\top + \U \theta_B \V^\top + \U \B \theta_V^\top, \theta_U \B \V^\top + \U \theta_B \V^\top + \U \B \theta_V^\top] + 2\langle \nabla f(\U\B\V^\top), \theta_U \B \theta_V^\top \rangle \\
& \overset{ \text{Proposition } \ref{prop: general-bijection2}, \eqref{eq: Hessian-con-gradient-general2} } =   \nabla^2 f(\X)[\cL^r_{\U,\B,\V}(\theta_{(\U,\B,\V)}),\cL^r_{\U,\B,\V}(\theta_{(\U,\B,\V)})] + 2\langle \nabla f(\X), P_{\U_\perp} \theta_U \B \B^{-1} \B \theta_V^\top  P_{\V_\perp}    \rangle\\
& = \Hess f(\X)[\cL^r_{\U,\B,\V}(\theta_{(\U,\B,\V)}),\cL^r_{\U,\B,\V}(\theta_{(\U,\B,\V)})],
\end{split}
\end{equation*}
where the last equality follows from the expression of $\Hess f(\X)$ in (<ref>) and the definition of $\cL^r_{\U,\B,\V}$.

Then, by (<ref>), (<ref>) and Theorem <ref>, we have $\overline{\Hess \, h_{r}([\U,\B,\V])}$ has $(p_1+p_2- r)r$ eigenvalues and  $\widebar{\lambda}_i(\Hess \, h_{r}([\U,\B,\V]))$ is sandwiched between $\sigma^2_r(\X)\lambda_i(\Hess f(\X)) $ and $2\sigma^2_1(\X) \lambda_i(\Hess f(\X)) $ for $i = 1,\ldots,(p_1+p_2- r)r$.   $\blacksquare$

Suppose the conditions in Proposition <ref> hold and the $\V_\Y,\W_\Y$ in $\bar{g}^r_{(\U,\Y)}$ satisfies $\V_\Y = \O \V_{\Y\O} \O^\top$ and $\W_\Y = \O \W_{\Y\O} \O^\top$ for any $\O \in \bbO_r$. Then

\begin{equation} \label{eq: gradient-connect-general3}
\begin{split}
\grad f(\X) &= \overline{\grad_\U\, h_{r}([\U,\Y])}\V_\Y \Y^\dagger  + \left(\overline{\grad_\Y\, h_{r}([\U,\Y])} \W_\Y \U^\top \right)^\top  \\
\overline{\grad\, h_{r}([\U,\Y])} &= \begin{bmatrix}
P_{\U_\perp}\grad f(\X) \Y \V_\Y^{-1} \\
(\grad f(\X))^\top \U \W_\Y^{-1}
\end{bmatrix}.
\end{split}
\end{equation}

Furthermore, if $[\U,\Y]$ is a Riemannian FOSP of (<ref>), we have:

\begin{equation} \label{eq: Hessian-connection-general3}
\begin{split}
\overline{\Hess \, h_{r}([\U,\Y])}[\theta_{(\U,\Y)}, \theta_{(\U,\Y)}]  &= \Hess f(\X)[\cL_{\U,\Y}^{r}(\theta_{(\U,\Y)}),\cL_{\U,\Y}^{r}(\theta_{(\U,\Y)})], \quad \forall \theta_{(\U,\Y)} \in \cH_{(\U,\Y)} \widebar{\cM}_{r}^{q_3}.
\end{split}
\end{equation}

Finally, $\overline{\Hess \, h_{r}([\U,\Y])}$ has $(p_1 + p_2-r)r$ eigenvalues and for $i = 1,\ldots, (p_1 + p_2-r)r$, we have
$\lambda_i(\overline{\Hess \, h_{r}([\U,\Y])})$ is sandwiched between $(\sigma_r(\W^{-1}_\Y) \wedge \sigma_r^2(\Y \V_\Y^{-1/2} )) \lambda_i(\Hess f(\X)) $ and $( \sigma_1(\W_\Y^{-1}) \vee \sigma^2_1(\Y\V_\Y^{-1/2})   ) \lambda_i(\Hess f(\X)) $.

Proof of Theorem <ref>. The proof is similar to the proof of Theorem <ref> and is postponed to Appendix <ref>.   $\blacksquare$

In Table <ref>, we provide explicit values of gap coefficients of the sandwich inequalities in Theorems <ref>, <ref> and <ref> under several different metrics specified by $(\W_{\L,\R}, \V_{\L,\R})$, $(\V_\B,\W_\B)$ and $(\V_\Y, \W_\Y)$ in $\bar{g}^{r}_{(\L,\R)}$, $\bar{g}^{r}_{(\U,\B,\V)}$, and $\bar{g}^{r}_{(\U,\Y)}$, respectively.

215emChoices of $(\W_{\L,\R}, \V_{\L,\R})$, $(\V_\B, \W_\B)$, and $(\V_\Y, \W_\Y)$ in $\bar{g}^{r}$     28emGap Coefficient Lower Bound     28emGap Coefficient Upper Bound

27em$\cM_{r}^e$ v.s. $\cM^{q_1}_{r1}$     $\W_{\L,\R} = (\L^\top \L)^{-1}$,  $\V_{\L,\R} = (\R^\top \R)^{-1}$      $\sigma^2_r(\X)$      $2 \sigma^2_1(\X)$

$\W_{\L,\R} = \R^\top \R$,  $\V_{\L,\R} = \L^\top \L$     $1$     $2$

17em$\cM_{r}^e$ v.s. $\cM^{q_2}_{r}$     $\V_\B = \I_r$, $\W_\B = \B^{-1}$      $\sigma^2_r(\X)$      $ 2\sigma^2_1(\X)$

37em$\cM_{r}^e$ v.s. $\cM^{q_3}_{r}$     $\V_\Y = \I_r$, $\W_\Y = \I_r$      $\sigma^2_r(\X) \wedge 1 $      $\sigma^2_1(\X) \vee  1$

$\V_\Y = \I_r, \W_\Y = (\Y^\top \Y)^{-1}$      $ \sigma^2_r(\X) $      $\sigma^2_1(\X)$

$\V_\Y = \Y^\top \Y, \W_\Y = \I_r$      $1 $      $1$

Gap coefficients in the sandwich inequalities in Theorems <ref>, <ref>, and <ref> under different Riemannian metrics.

By Theorems <ref>, <ref> and <ref>, we have the following Corollary <ref> on the equivalence of Riemannian FOSPs, SOSPs and strict saddles of optimization (<ref>) under the embedded and the quotient geometries. The proof is given in Appendix <ref>.
(Equivalence on Riemannian FOSPs, SOSPs and strict saddles of (<ref>) Under Embedded and Quotient Geometries)  Suppose $\W_{\L,\R} = \M \W_{\L\M,\R \M^{-\top}} \M^\top$, $\V_{\L,\R} = \M^{-\top } \V_{\L\M,\R \M^{-\top}} \M^{-1}$ hold for any $\M \in \GL(r)$ and $\V_\Y = \O \V_{\Y\O} \O^\top$, $\W_\Y = \O \W_{\Y\O} \O^\top$ holds for any $\O \in \bbO_r$. Then we have

(a) given $\L \in \bbR^{p_1 \times r}_*, \R \in \bbR^{p_2 \times r}_*, \U \in \st(r,p_1)$, $\B \in \bbS_{+}(r)$, $\V \in \st(r,p_2)$ and $\Y \in \bbR^{p_2 \times r}_*$, if $[\L,\R]$ ($[\U,\B,\V]$ or $[\U,\Y]$) is a Riemannian FOSP or SOSP or strict saddle of (<ref>) ((<ref>) or (<ref>)), then $\X = \L \R^\top$ ($\X = \U \B \V^\top$ or $\X = \U \Y^\top$) is a Riemannian FOSP or SOSP or strict saddle of (<ref>) under the embedded geometry;

(b)if $\X$ is a Riemannian FOSP or SOSP or strict saddle of (<ref>) under the embedded geometry, then there is a unique $[\L,\R]$ ($[\U,\B,\V]$ or $[\U,\Y]$) such that $\L \R^\top = \X$ ($\U \B \V^\top = \X$ or $\U \Y^\top =\X$) and it is a Riemannian FOSP or SOSP or strict saddle of (<ref>) ((<ref>) or (<ref>)).

(Geometric Connection of Non-convex Factorization and Quotient Manifold Formulations for (<ref>) and (<ref>))
As we have discussed in the Introduction, another popular approach for handling the rank constraint in (<ref>) or (<ref>) is via factorizing $\X$ into $\Y \Y^\top$ or $\L \R^\top$ and then treating the new problem as unconstrained optimization in the Euclidean space. In the recent work <cit.>, they showed for both (<ref>) and (<ref>) the geometric landscapes under the factorization and embedded submanifold formulations are almost equivalent. By combining their results and the results in this paper, we also have a geometric landscape connection of (<ref>) and (<ref>) under the factorization and quotient manifold formulations.

(Algorithmic Connection of Embedded and Quotient Geometries)

In contrast to the geometric connections of an optimization problem under two different geometries, the relationship of two sequences of iterates generated by algorithms in two geometries is more subtle. First, the algorithms under the quotient geometry are performed in the horizontal space and they depend on the quotient structure and the Riemannian metric we pick. Thus, it is hard to expect a universal algorithmic connection under two geometries. On the other hand, with some careful calculation, we find that by taking some specific metrics under the quotient geometry, there are indeed some connections of gradient flows, i.e., solutions of dynamical systems $d \X /d t = -\grad f(\X)$, $d [\Z] /d t = -\grad h([\Z])$ <cit.>, produced under the embedded and quotient geometries. This is particularly true when the metrics are chosen in a way such that the sandwich gap coefficients in the geometric connection are some universal constants (see Remark <ref> and Tables <ref> and <ref>). Next, we illustrate a few such examples of gradient flows under various embedded and quotient geometries. Specifically, we compare the dynamical system $d \X /d t = -\grad f(\X)$ under the embedded geometry and the induced dynamical system $d \ell([\Z]) /dt = - \rmD \ell([\Z])[ \grad h([\Z]) ] = - \rmD  \bar{\ell}(\Z) [ \overline{\grad h([\Z])}  ]$ from the quotient geometry with the link $\X = \ell([\Z])$:

* (PSD case) Take $\W_\Y = 2\Y^\top \Y$ in $\bar{g}_\Y^{r+}$ and $\V_\B = 2\B^2, \W_\B = \I_r$ in $\bar{g}_{(\U,\B)}^{r+}$, we have the following gradient flows of $\{ \X_t \}$ in (<ref>) under $\cM_{r+}^e$, $\cM_{r+}^{q_1}$ and $\cM_{r+}^{q_2}$:

\begin{equation*}
\begin{split}
\text{under } \cM_{r+}^e: &\quad d \X /d t = -\grad f(\X) = -P_{\U} \nabla f(\X) - \nabla f(\X)P_{\U} + P_{\U} \nabla f(\X)P_{\U},\\
\text{under } \cM_{r+}^{q_1}:& \quad d \ell([\Z]) /d t = -\overline{\grad\, h_{r+}([\Y])}\Y^\top - \Y \left(\overline{\grad\, h_{r+}([\Y])}\right)^\top = -P_{\U} \nabla f(\X) - \nabla f(\X)P_{\U};\\
\text{under } \cM_{r+}^{q_2}:& \quad d \ell([\Z]) / dt = - \overline{\grad_\U\, h_{r+}([\U,\B])} \B \U^\top - \U \overline{\grad_\B\, h_{r+}([\U,\B])} \U^\top \\
& \quad \quad \quad \quad \quad \quad  - \U \B (\overline{\grad_\U\, h_{r+}([\U,\B])})^\top \\
& \quad \quad \quad \quad \quad \,  = -P_{\U} \nabla f(\X) - \nabla f(\X)P_{\U} + P_{\U} \nabla f(\X)P_{\U}.
\end{split}
\end{equation*}

* (general case) Take $\W_{\L,\R} = \R^\top \R, \V_{\L,\R} = \L^\top \L$ in $\bar{g}^r_{(\L,\R)}$, $\V_\Y = \Y^\top \Y, \W_\Y = \I_r$ in $\bar{g}^r_{(\U,\Y)}$, we have the following gradient flows of $\{\X_t \}$ in (<ref>) under $\cM_{r}^e$, $\cM_{r}^{q_1}$ and $\cM_{r}^{q_3}$:

\begin{equation*}
\begin{split}
\text{under } \cM_{r}^e: &\quad d \X /d t = -\grad f(\X) = -P_{\U} \nabla f(\X) - \nabla f(\X)P_{\V} + P_{\U} \nabla f(\X)P_{\V},\\
\text{under } \cM_{r}^{q_1}: & \quad d \ell([\Z]) /d t = -\overline{\grad_{\L}\, h_{r}([\L,\R])}\R^\top - \L \left(\overline{\grad_{\R}\, h_{r}([\L,\R])}\right)^\top = -P_{\U} \nabla f(\X) - \nabla f(\X)P_{\V},\\
\text{under } \cM_{r}^{q_3}: & \quad d \ell([\Z]) /d t = - \overline{\grad_\U\, h_{r}([\U,\Y])} \Y^\top - \U (\overline{\grad_\Y\, h_{r}([\U,\Y])})^\top \\
& \quad \quad \quad \quad \quad  = -P_{\U} \nabla f(\X) - \nabla f(\X)P_{\V} + P_{\U} \nabla f(\X)P_{\V}.
\end{split}
\end{equation*}

Consider $\cM_{r+}^e$ v.s. $\cM_{r+}^{q_1}$ or $\cM_{r}^e$ v.s. $\cM_{r}^{q_1}$ with the specific metrics chosen above, we can see that the gradient flows of (<ref>) or (<ref>) under these embedded and the quotient geometries only differ one term, which has a magnitude smaller than the other terms. In addition, we know from Tables <ref> and <ref> that the sandwich coefficients lower and upper bounds are universal constants $1$ and $2$, respectively, in these settings. Some empirical evidence which shows the remarkably similar algorithmic performance under these embedded and quotient geometries was provided in <cit.> and here our geometric connection results provide more theoretical insights for this empirical observation. More interestingly, we discover the gradient flows under embedded and quotient geometries are identical when the spectrum of Riemannian Hessians under two geometries coincide at Riemannian FOSPs, e.g., see $\cM_{r+}^e$ v.s. $\cM_{r+}^{q_2}$ or $\cM_{r}^e$ v.s. $\cM_{r}^{q_3}$ with the specific metrics chosen above.

§ CONCLUSION AND DISCUSSIONS

In this paper, we propose a general procedure for establishing geometric connections of a Riemannian optimization problem under embedded and quotient geometries. By applying the general procedure to problems (<ref>) and (<ref>) under the embedded and quotient geometries, we establish an exact Riemannian gradient connection under two geometries at every point on the manifold and sandwich inequalities between the spectra of the Riemannian Hessians at Riemannian FOSPs. These results immediately imply an equivalence on the sets of Riemannian FOSPs, SOSPs, and strict saddles of (<ref>) and (<ref>) under embedded and quotient geometries. Moreover, we observe an intriguing algorithmic connection for fixed-rank matrix optimization under two geometries with some specific Riemannian metrics. After the first release of this manuscript, there are a couple of new related works that are worth mentioning. In <cit.>, the authors studied the landscape connection of a pair of optimization problems, including a related bounded-rank matrix optimization problem, connected by a smooth parameterization. <cit.> studied the algorithmic connection of Riemannian gradient descent type algorithms under the embedded geometry and a few quotient geometries in the PSD fixed-rank optimization problem. These works provide a good complement to the results in this paper.

There are many interesting extensions to the results in this paper to be explored in the future. First, as we have mentioned in the Example <ref> in Section <ref>, our results on the connection of Riemannian Hessians under the embedded and the quotient geometries are established at Riemannian FOSPs. It is interesting to explore whether it is possible to connect the landscapes under two geometries at non-stationary points. Second, other than the geometries covered in this paper, there are many other embedded and quotient geometries for fixed-rank matrices, such as the ones in <cit.> for $\cM_{r+}$, it will be interesting to study the geometric landscape connection of (<ref>) and (<ref>) under these geometries. Third, another common manifold that has both embedded and quotient representations is the Stiefel manifold <cit.>. We believe our general procedure in Section <ref> can also be used to establish the geometric connection in that setting as well. Finally, our ultimate goal is to better understand the connections and comparisons of different Riemannian geometries and give some guidelines on how to choose them given a Riemannian optimization problem. Some progress and discussions on how to choose Riemannian metrics in quotient geometries can be found in <cit.>. While there is still not too much study on how to choose different quotient structures and manifold classes. It is important to explore these directions in the future.

§ ACKNOWLEDGMENTS.
The authors would like to thank the editors and two reviewers for their great suggestions and comments. Y. Luo would like to thank Nicolás García Trillos for helpful discussions during the project.

§ ADDITIONAL PRELIMINARIES ON RIEMANNIAN CONNECTION

In this section, we provide a more detailed discussion of the Riemannian connection as it is critical in deriving the Riemannian Hessians. First, a vector field $\xi$ on a manifold $\cM$ is a mapping that assigns each point $\X \in \cM$ a tangent vector $\xi_\X \in T_\X \cM$. The Riemannian gradient is a typical example of the vector field.

The coordinate-free definition of the Riemannian connection follows from the fundamental theorem of differential geometry <cit.> and can be characterized by the so-called Koszul formula given as follows:

\begin{equation} \label{eq: general-koszul-formula}
2g_\X (\widebar{\nabla}_\xi \eta, \theta ) = {\rm D} g_\X (\eta,\theta)[\xi] + \rmD g_\X(\xi,\theta)[\eta] - \rmD g_\X(\eta, \xi)[\theta] + g_\X( \theta, [\xi, \eta] ) + g_\X (\eta,[\theta,\xi]) - g_\X (\xi,[\eta, \theta]),
\end{equation}
where $[\xi,\eta]$ denotes the Lie bracket between vector fields $\xi$ and $\eta$. If $\cM$ is an open subset of a vector space, the Lie bracket of vector fields is given by

\begin{equation} \label{eq: lie-bracket}
[\xi, \eta] = \rmD \eta [\xi] - \rmD \xi [\eta],
\end{equation}

where $\rmD \eta [\xi]$ is the classical direction derivative of a vector field in a vector space and its evaluation at $\X$ is denoted by $(\rmD \eta [\xi])_\X$ and given as follows: $(\rmD \eta [\xi])_\X = \lim_{t \to 0} ( \eta_{\X + t \xi_\X} - \eta_\X )/t $ <cit.>.

In the following Proposition <ref>, we present the Koszul formulas associated with each simple manifold in Table <ref> and their combinations. These formulas will be used in deriving Riemannian Hessians under the quotient geometries given in Sections <ref> and <ref>.
(Koszul Formulas Associated with $\bbR^{p \times r}_*$, $\st(r,p)$, $\bbS_{+}(r)$ and their combinations)

(i) (Koszul Formulas Associated with $\bbR^{p \times r}_*$, $\st(r,p)$ and $\bbS_{+}(r)$.) Consider manifolds $\bbR^{p \times r}_*$, $\st(r,p)$ and $\bbS_{+}(r)$ endowed metrics $g$ given in Table <ref> with $\V_\lozenge = \I$ and $\W_\B = \B^{-1}$ . For vector fields $\xi, \eta, \theta$ on $\bbR^{p \times r}_*$ or $\st(r,p)$ or $\bbS_{+}(r)$, we have the following Koszul formulas:

\begin{equation} \label{eq: koszul-formula-simple-manifolds}
\begin{split}
\bbR^{p \times r}_*: \, 2 g_\Y ( \widebar{\nabla}_\xi \eta, \theta  ) &= 2 g_\Y ( \rmD \eta[\xi], \theta ) + \tr( \rmD \W_\Y [\xi] \eta^\top \theta ) + \tr( \rmD \W_\Y [\eta] \xi^\top \theta ) - \tr( \rmD \W_\Y [\theta] \eta^\top \xi ),\\
\st(r,p): \,	2 g_\U (\widebar{\nabla}_\xi \eta, \theta  ) & = 2 g_\U (\rmD \eta[\xi], \theta ),\\
\bbS_{+}(r): \,	2 g_\B (\widebar{\nabla}_\xi \eta, \theta ) & = 2 g_\B (\rmD \eta[\xi], \theta) - \tr(\B^{-1} \xi \B^{-1} \eta \B^{-1} \theta ) - \tr(\B^{-1} \eta \B^{-1}  \xi \B^{-1} \theta ).
\end{split}
\end{equation}
Here, the operation on vector fields is performed elementwise on the associated tangent vectors. For example, $\eta^\top \theta$ is mapping that assign each point $\Y \in \bbR^{p \times r}_*$ to the matrix $\eta_\Y^\top \theta_\Y$, i.e., $(\eta^\top \theta)_\Y = \eta_\Y^\top \theta_\Y$.

(ii) (Koszul Formula Associated with $\st(r,p)\times \bbS_{+}(r)$.)
Suppose we have metric $g_{(\U, \B)}$ on $\st(r,p)\times \bbS_{+}(r)$ given by $g_{(\U,\B)}(\eta_{(\U, \B)}, \theta_{(\U, \B)}) = \tr(\V_\B \eta_U^\top \theta_U) + \tr(\W_\B \eta_B \W_\B \theta_B)$ for $\eta_{(\U, \B)} = [\eta_U^\top \quad \eta_B^\top]^\top, \theta_{(\U, \B)} = [\theta_U^\top \quad \theta_B^\top]^\top \in T_\U \st(r,p) \times T_\B \bbS_+(r)$. Then for vector fields $\xi, \eta, \theta$ on $\st(r,p)\times \bbS_{+}(r)$,

\begin{equation}\label{eq: koszul-formula-st-s+}
\begin{split}
2g_{(\U, \B)} ( \widebar{\nabla}_{\xi_{(\U', \B')}} \eta, \theta_{(\U', \B')} )&= 2 g_{(\U, \B)}( \rmD \eta_{(\U', \B')}[\xi_{(\U', \B')}], \theta_{(\U', \B')} ) \\
& + \tr( \rmD \V_\B[\xi_{B'}] \eta_{U'}^\top \theta_{U'} ) + 2\tr( \sym( \rmD \W_\B[\xi_{B'}] \eta_{B'} \W_\B) \theta_{B'}   ) \\
& + \tr( \rmD \V_\B[\eta_{B'}] \xi_{U'}^\top \theta_{U'} ) + 2\tr( \sym( \rmD \W_\B[\eta_{B'}] \xi_{B'} \W_\B   ) \theta_{B'}   )\\
& - \tr( \rmD \V_\B[\theta_{B'}] \eta_{U'}^\top \xi_{U'} ) - 2\tr( \sym( \rmD \W_\B[\theta_{B'}] \eta_{B'} \W_\B ) \xi_{B'}   ).
\end{split}
\end{equation}

(iii) (Koszul Formula Associated with $\st(r,p_1) \times \bbR^{p_2 \times r}_*$.)  Suppose we have metric $g_{(\U, \Y)}$ on $\st(r,p_1) \times \bbR^{p_2 \times r}_*$ given by $g_{(\U,\Y)}(\eta_{(\U, \Y)}, \theta_{(\U, \Y)}) = \tr(\V_\Y \eta_U^\top \theta_U) + \tr(\W_\Y \eta_Y^\top \theta_Y)$ for $\eta_{(\U, \Y)} = [\eta_U^\top \quad \eta_Y^\top]^\top, \theta_{(\U, \Y)} = [\theta_U^\top \quad \theta_Y^\top]^\top \in T_\U \st(r,p_1) \times T_\Y \bbR^{p_2 \times r}_*$. Then for vector fields $\xi, \eta, \theta$ on $\st(r,p_1) \times \bbR^{p_2 \times r}_*$,

\begin{equation} \label{eq: koszul-formula-st-R}
\begin{split}
2g_{(\U, \Y)} ( \widebar{\nabla}_{\xi_{(\U', \Y')}} \eta, \theta_{(\U', \Y')} )&= 2 g_{(\U, \Y)}( \rmD \eta_{(\U', \Y')}[\xi_{(\U', \Y')}], \theta_{(\U', \Y')} ) \\
& + \tr( \rmD \V_\Y[\xi_{Y'}] \eta_{U'}^\top \theta_{U'} ) + \tr( \rmD \W_\Y [\xi_{Y'}] \eta_{Y'}^\top \theta_{Y'}  ) \\
& + \tr( \rmD \V_\Y[\eta_{Y'}] \xi_{U'}^\top \theta_{U'} ) + \tr( \rmD \W_\Y [\eta_{Y'}] \xi_{Y'}^\top \theta_{Y'}  )\\
& - \tr( \rmD \V_\Y[\theta_{Y'}] \eta_{U'}^\top \xi_{U'} ) - \tr( \rmD \W_\Y [\theta_{Y'}] \eta_{Y'}^\top \xi_{Y'}  ).
\end{split}
\end{equation}

Proof of Proposition <ref>. (Part (i)) The Koszul formulas associated with $\st(r,p)$ and $\bbS_{+}(r)$ can be found in <cit.> and <cit.>, respectively. Next, we derive the Koszul formula associated with the $\bbR^{p \times r}_*$ by using the general formula (<ref>). First, for any vector fields $\eta, \theta, \xi$ on $\bbR^{p \times r}_*$, we have

\begin{equation} \label{eq: directional-derivative-gY-instance}
\begin{split}
(\rmD g_\Y (\eta, \theta )[\xi])_{\Y'} &= \lim_{t \to 0} \frac{\tr( \W_{\Y + t\xi_{\Y'} } \eta_{\Y' + t \xi_{\Y'} }^\top \theta_{\Y' + t\xi_{\Y'} }  )  - \tr( \W_\Y \eta_{\Y'}^\top \theta_{\Y'} ) }{t} \\
&=  \lim_{t \to 0} \frac{\tr( \W_{\Y + t\xi_{\Y'} } \eta_{\Y' + t \xi_{\Y'} }^\top \theta_{\Y' + t\xi_{\Y'} }  )  - \tr( \W_\Y \eta_{\Y' + t \xi_{\Y'} }^\top \theta_{\Y' + t\xi_{\Y'} }  ) }{t} \\
& \quad + g_\Y( \rmD \eta_{\Y'}[\xi_{\Y'}], \theta_{\Y'} ) + g_\Y( \eta_{\Y'}, \rmD \theta_{\Y'}[\xi_{\Y'}] ) \\
& = \tr(\rmD \W_\Y [\xi_{\Y'}] \eta_{\Y'}^\top \theta_{\Y'} ) + g_\Y( \rmD \eta_{\Y'}[\xi_{\Y'}], \theta_{\Y'} ) + g_\Y( \eta_{\Y'}, \rmD \theta_{\Y'}[\xi_{\Y'}] ).
\end{split}
\end{equation}
This proves

\begin{equation} \label{eq: directional-derivative-gY}
\rmD g_\Y (\eta, \theta)[\xi] = \tr(\rmD \W_\Y [\xi] \eta^\top \theta ) + g_\Y( \rmD \eta[\xi], \theta ) + g_\Y( \eta, \rmD \theta[\xi] ).
\end{equation}

Then by (<ref>) and the fact $\bbR^{p \times r}_*$ is an open subset of $\bbR^{p \times r}$, we have

\begin{equation} \label{eq: koszul-formula-derive-gY}
\begin{split}
&\quad 2 g_\Y (\widebar{\nabla}_\xi \eta, \theta) \\
&= {\rm D} g_\Y (\eta,\theta)[\xi] + \rmD g_\Y(\xi,\theta)[\eta] - \rmD g_\Y(\eta, \xi)[\theta] + g_\Y( \theta, [\xi, \eta] ) + g_\Y (\eta,[\theta,\xi]) - g_\Y (\xi,[\eta, \theta])\\
& \overset{ \eqref{eq: lie-bracket},\eqref{eq: directional-derivative-gY}} = \tr(\rmD \W_\Y [\xi] \eta^\top \theta ) + g_\Y( \rmD \eta[\xi], \theta ) + g_\Y( \eta, \rmD \theta[\xi] ) + \tr(\rmD \W_\Y [\eta] \xi^\top \theta ) + g_\Y( \rmD \xi[\eta], \theta ) \\
& \quad   + g_\Y( \xi, \rmD \theta[\eta] ) - \tr(\rmD \W_\Y [\theta] \eta^\top \xi ) - g_\Y( \rmD \eta[\theta], \xi ) - g_\Y( \eta, \rmD \xi[\theta] ) + g_\Y(\theta, \rmD \eta[\xi] - \rmD \xi[\eta]) \\
& \quad + g_\Y(\eta, \rmD \xi[\theta] - \rmD \theta[\xi]) - g_\Y(\xi, \rmD \theta[\eta] - \rmD \eta[\theta])\\
& = 2 g_\Y ( \rmD \eta[\xi], \theta ) + \tr( \rmD \W_\Y [\xi] \eta^\top \theta ) + \tr( \rmD \W_\Y [\eta] \xi^\top \theta ) - \tr( \rmD \W_\Y [\theta] \eta^\top \xi ).
\end{split}
\end{equation}

(Part (ii)). For vector fields $\xi, \eta, \theta$ on $\st(r,p)\times \bbS_{+}(r)$,

\begin{equation}\label{eq: directional-derivative-gUB-instance}
\begin{split}
&\rmD g_{(\U, \B)}(\eta_{(\U', \B')}, \theta_{(\U', \B')})[\xi_{(\U', \B')}] \\
=& \lim_{t \to 0} \frac{ \substack{\tr(\V_{\B + t \xi_{B'}} \eta_{U' + t \xi_{U'} }^\top \theta_{U' + t\xi_{U'} }) + \tr(\W_{\B + t \xi_{B'}} \eta_{B' + t \xi_{B'} } \W_{\B + t \xi_{B'}} \theta_{B'+ t \xi_{B'}}) - \tr(\V_\B \eta_{U'}^\top \theta_{U'}) - \tr(\W_\B \eta_{B'} \W_\B \theta_{B'}) }}{t} \\
= & \tr( \rmD \V_\B[ \xi_{B'} ] \eta_{U'}^\top \theta_{U'} ) + \tr( \rmD \W_{\B}[\xi_{B'}] \eta_{B'} \W_\B \theta_{B'} )   + \tr( \W_\B \eta_{B'} \rmD \W_{\B}[\xi_{B'}] \theta_{B'} )\\
& + \tr( \V_\B \rmD \eta_{U'} [ \xi_{U'} ]^\top \theta_{U'} )  + \tr( \V_\B \eta_{U'}\rmD \theta_{U'}[ \xi_{U'} ] ) + \tr( \W_\B \rmD \eta_{B'} [\xi_{B'} ]  \W_\B \theta_{B'} ) + \tr( \W_\B \eta_{B'}  \W_\B \rmD \theta_{B'} [\xi_{B'} ]  ) \\
= & g_{(\U, \B)}(  \rmD \eta_{(\U', \B')}[\xi_{(\U', \B')}], \theta_{(\U', \B')}  ) + g_{(\U, \B)}(  \eta_{(\U', \B')}, \rmD \theta_{(\U', \B')}[\xi_{(\U', \B')}] ) \\
& +  \tr( \rmD \V_\B[ \xi_{B'} ] \eta_{U'}^\top \theta_{U'} ) + \tr( \rmD \W_{\B}[\xi_{B'}] \eta_{B'} \W_\B \theta_{B'} )   + \tr( \W_\B \eta_{B'} \rmD \W_{\B}[\xi_{B'}] \theta_{B'} ).
\end{split}
\end{equation}
Combining (<ref>) with (<ref>), we conclude (<ref>).

(Part (iii)).  For vector fields $\xi, \eta, \theta$ on $\st(r,p_1) \times \bbR^{p_2 \times r}_*$, we have

\begin{equation}\label{eq: directional-derivative-gUY-instance}
\begin{split}
& \rmD g_{(\U, \Y)}(\eta_{(\U', \Y')}, \theta_{(\U', \Y')})[\xi_{(\U', \Y')}] \\
= &  \lim_{t \to 0} \frac{\tr(\V_{\Y + t \xi_{Y'}} \eta_{U' + t \xi_{U'} }^\top \theta_{U' + t\xi_{U'} }) + \tr(\W_{\Y + t \xi_{Y'}} \eta_{Y' + t \xi_{Y'} }^\top \theta_{Y'+ t \xi_{Y'}})  - \tr(\V_\Y \eta_{U'}^\top \theta_{U'}) - \tr(\W_\Y \eta_{Y'}^\top \theta_{Y'}) }{t}\\
= & \tr( \rmD \V_\Y[\xi_{Y'}] \eta_{U'}^\top \theta_{U'} )   + \tr(\rmD \W_\Y[\xi_{Y'}] \eta_{Y'}^\top \theta_{Y'}) \\
& + g_{(\U,\Y)}( \rmD\eta_{(\U', \Y')} [\xi_{(\U', \Y')} ], \theta_{(\U', \Y')}  )  + g_{(\U,\Y)}(\eta_{(\U', \Y')}, \rmD\theta_{(\U', \Y')}[\xi_{(\U', \Y')} ]  ).
\end{split}
\end{equation}
Combining (<ref>) with (<ref>), we conclude (<ref>). This finishes the proof.   $\blacksquare$.

§ ADDITIONAL PROOFS IN SECTION <REF>

§.§ Proof of Lemma <ref>.

Since $\ell$ is a diffeomorphism between $\cM^q$ and $\cM^e$, $\ell$ and $\ell^{-1}$ are both open maps <cit.>. If $\X$ is a minimizer of (<ref>) in a neighborhood of $\X$ say $\cN_1$, $\ell^{-1}$ maps $\cN_1$ to an open neighborhood of $\ell^{-1}(\X)$ say $\cN_2$, and $\ell^{-1}(\X)$ is a minimizer of (<ref>) for all $[\Z]\in \cN_2$. This implies $\ell^{-1}(\X)$ is a local minimizer of (<ref>). This argument also applies in another direction.   $\blacksquare$

§.§ Proof of Theorem <ref>

First, because $\overline{\Hess\, h([\Z])}$ and $\Hess f(\X)$ are by definition self-adjoint linear maps from $\cH_\Z \widebar{\cM}^q$ and $T_\X \cM^e$ to $\cH_\Z \widebar{\cM}^q$ and $T_\X \cM^e$, respectively, both $\Hess f(\X)$ and $\overline{\Hess\, h([\Z])}$ have $p$ eigenvalues as $\dim(T_\X \cM^e) = \dim(\cH_\Z \widebar{\cM}^q) = d$. Suppose $\u_1,\ldots,\u_d$ are eigenvectors corresponding to $\lambda_1(\Hess f(\X))$,$\ldots$, $\lambda_p(\Hess f(\X))$ and $\mathbf{v}_1,\ldots,\mathbf{v}_d$ are eigenvectors corresponding to $\lambda_1(\overline{\Hess\, h([\Z])}),\ldots,\lambda_p(\overline{\Hess\, h([\Z])})$. For $k = 1,\ldots, d$, define

\begin{equation*}
\begin{split}
&\mathcal{U}_k = \rspan\{\mathbf{u}_1,\ldots,\mathbf{u}_k\}, \quad \mathcal{U}'_k = \rspan\{\cL^{-1}(\mathbf{u}_1),\ldots,\cL^{-1}(\mathbf{u}_k)\},\\
&\mathcal{V}_k = \rspan\{\mathbf{v}_1,\ldots,\mathbf{v}_k\}, \quad \mathcal{V}'_k = \rspan\{\cL(\mathbf{v}_1),\ldots,\cL(\mathbf{v}_k)\}.
\end{split}
\end{equation*}

Let us first consider the case that $\lambda_k(\Hess f(\X)) \geq 0$. The Max-min theorem for eigenvalues (Lemma <ref>) yields

\begin{equation} \label{ineq: spectrum-ineq1}
\begin{split}
\lambda_k(\overline{\Hess\, h([\Z])}) &\geq \min_{\mathbf{u}' \in \cU_k', \mathbf{u}' \neq 0  } \frac{\overline{\Hess\, h([\Z])}[\mathbf{u}',\mathbf{u}']}{\bar{g}_\Z(\mathbf{u}',\mathbf{u}')} \overset{ \eqref{eq: general-setting-hessian-connection} }= \min_{\mathbf{u}' \in \cU_k', \mathbf{u}' \neq 0} \frac{\Hess f(\X)[\cL(\mathbf{u}'),\cL(\mathbf{u}')]}{\bar{g}_\Z(\mathbf{u}',\mathbf{u}')} \\
& =  \min_{\mathbf{u} \in \cU_k, \mathbf{u} \neq 0} \frac{\Hess f(\X)[\mathbf{u},\mathbf{u}]}{\bar{g}_\Z(\cL^{-1}(\mathbf{u}),\cL^{-1}(\mathbf{u}))} \geq  \min_{\mathbf{u} \in \cU_k,\mathbf{u} \neq \0 }\frac{\lambda_k(\Hess f(\X))g_\X(\mathbf{u},\mathbf{u})}{\bar{g}_\Z(\cL^{-1}(\mathbf{u}),\cL^{-1}(\mathbf{u}))} \\
&\overset{\eqref{eq: spectrum-bound-general-setting-L} } \geq \alpha \lambda_k(\Hess f(\X)) \geq 0.
\end{split}
\end{equation}
On the other hand, we have

\begin{equation}\label{ineq: spectrum-ineq2}
\begin{split}
& \quad \lambda_k(\Hess f(\X)) \\
&\overset{\text{Lemma } \ref{lm: max-min-theorem}} \geq \min_{\mathbf{v}' \in \cV_k',\mathbf{v}' \neq \0 } \frac{\Hess f(\X)[\cL \cL^{-1}(\mathbf{v}'),\cL \cL^{-1}(\mathbf{v}')]}{g_\X(\mathbf{v}',\mathbf{v}')} \overset{ \eqref{eq: general-setting-hessian-connection} } = \min_{\mathbf{v}' \in \cV_k',\mathbf{v}' \neq \0 } \frac{\overline{\Hess\, h([\Z])}[\cL^{-1}(\mathbf{v}'),\cL^{-1}(\mathbf{v}')]}{g_\X(\mathbf{v}',\mathbf{v}')}\\
&= \min_{\mathbf{v} \in \cV_k,\mathbf{v} \neq \0 } \frac{\overline{\Hess\, h([\Z])}[\mathbf{v},\mathbf{v}] }{g_\X(\cL(\mathbf{v}),\cL(\mathbf{v}))} \geq \min_{\mathbf{v} \in \cV_k,\mathbf{v} \neq \0 } \frac{\lambda_k(\overline{\Hess\, h([\Z])}) \bar{g}_\Z(\mathbf{v},\mathbf{v}) }{g_\X(\cL(\mathbf{v}),\cL(\mathbf{v}))} \\
& \overset{\eqref{ineq: spectrum-ineq1}, \eqref{eq: spectrum-bound-general-setting-L} }\geq \lambda_k(\overline{\Hess\, h([\Z])} )/\beta.
\end{split}
\end{equation}
So we have proved the result for the case that $\lambda_k(\Hess f(\X)) \geq 0$. When $\lambda_k(\Hess f(\X)) < 0$, we have $\lambda_{p+1-k}(-\Hess f(\X)) = -\lambda_k(\Hess f(\X)) > 0$. Following the same proof of (<ref>) and (<ref>), we have

\begin{equation*}
\begin{split}
-\lambda_k(\overline{\Hess\, h([\Z])}) = \lambda_{p+1-k}(-\overline{\Hess\, h([\Z])}) \geq \alpha \lambda_{p+1-k}(-\Hess f(\X)) = -\alpha\lambda_k(\Hess f(\X)) > 0,\\
-\lambda_k(\Hess f(\X)) =  \lambda_{p+1-k}(-\Hess f(\X)) \geq  \lambda_{p+1-k}(-\overline{\Hess\, h([\Z])})/\beta = -\lambda_k(\overline{\Hess\, h([\Z])})/\beta.
\end{split}
\end{equation*}
This finishes the proof of this theorem.   $\blacksquare$

§ ADDITIONAL PROOFS IN SECTION <REF>

§.§ Proof of Lemma <ref>.

It has been shown in <cit.> that the vertical space of $\widebar{\cM}_{r+}^{q_1}$ is $\{ \theta_\Y: \theta_\Y = \Y \bOmega, \bOmega = - \bOmega^\top \}$ with dimension $(r^2-r)/2$. By observing $\Y = \U \P$, it is easy to check

\begin{equation*}
\begin{split}
\{ \theta_\Y: \theta_\Y = \U \bOmega' \P^{-\top}, \bOmega' = - \bOmega^{'\top} \} &=\{ \theta_\Y: \theta_\Y = \Y \P^{-1} \bOmega' \P^{-\top}, \bOmega' = - \bOmega^{'\top}  \} \\
& = \{ \theta_\Y: \theta_\Y = \Y \bOmega, \bOmega = - \bOmega^\top  \} = \cV_\Y \widebar{\cM}_{r+}^{q_1}.
\end{split}
\end{equation*}
In addition, notice $\dim(\cH_\Y \widebar{\cM}_{r+}^{q_1}) = pr-(r^2-r)/2$ and $(r^2-r)/2 + (pr-(r^2-r)/2) = pr$, then $\cH_\Y \widebar{\cM}_{r+}^{q_1}$ is the horizontal space if we can show it is orthogonal to $\cV_\Y \widebar{\cM}_{r+}^{q_1}$ with respect to $\bar{g}^{r+}_\Y$. For $\eta_\Y = \U \bOmega \P^{-\top} \in \cV_\Y \widebar{\cM}_{r+}^{q_1}$ and $\theta_\Y = (\U \S + \U_\perp \D) \P^{-\top} \in \cH_\Y \widebar{\cM}_{r+}^{q_1}$,

\begin{equation} \label{eq: lm-psd-quotient-horizontal-eq1}
\begin{split}
\bar{g}^{r+}_\Y(\eta_\Y, \theta_\Y) = \langle  \bOmega, \S \P^{-\top}\W_\Y \P^{-1} \rangle \overset{(a)}= - \langle  \bOmega^\top , \S \P^{-\top}\W_\Y \P^{-1} \rangle &\overset{(b)}= - \langle  \bOmega, \S \P^{-\top}\W_\Y \P^{-1} \rangle \\
&= - \bar{g}^{r+}_\Y(\eta_\Y, \theta_\Y),
\end{split}
\end{equation}
where (a) is because $\bOmega = - \bOmega^\top$ and (b) is because $\S \P^{-\top}\W_\Y \P^{-1}$ is symmetric by the construction of $\cH_\Y \widebar{\cM}_{r+}^{q_1}$. (<ref>) implies $\bar{g}^{r+}_\Y(\eta_\Y, \theta_\Y) = 0$. This finishes the proof for the first part.

From Section <ref>, we know to show $\cM_{r+}^{q_1}$ is a Riemannian quotient manifold endowed with the Riemannian metric $g^{r+}_{[\Y]}$ induced from $\bar{g}^{r+}_\Y$, it is enough to show $\bar{g}^{r+}_\Y (\eta_\Y, \theta_{\Y})= \bar{g}^{r+}_{\Y \O}(\eta_{\Y \O}, \theta_{\Y\O})$ for any $\O \in \bbO_r$, where $\eta_\Y, \theta_\Y$ and $\eta_{\Y \O}, \theta_{\Y\O}$ are the horizontal lifts of $\eta_{[\Y]}$ and $\theta_{[\Y]}$ at $\Y$ and $\Y\O$, respectively. By Lemma <ref>(i), we have the the horizontal lifts of $\eta_{[\Y]}$ at $\Y$ and $\Y \O$ are related as $\eta_{\Y\O} = \eta_{\Y} \O$. So

\begin{equation*}
\bar{g}^{r+}_{\Y \O}(\eta_{\Y \O}, \theta_{\Y\O}) = \tr(\W_{\Y\O} \eta_{\Y \O}^\top \theta_{\Y\O} ) = \tr(\W_{\Y\O} \O^\top \eta_\Y^\top \theta_\Y \O ),
\end{equation*}
and $\bar{g}^{r+}_{\Y \O}(\eta_{\Y \O}, \theta_{\Y\O}) = \bar{g}^{r+}_\Y (\eta_\Y, \theta_{\Y})$ holds for any $\eta_\Y, \theta_{\Y}$ if and only if $\W_\Y = \O \W_{\Y\O} \O^\top$ holds for any $\O \in \bbO_r$. This shows $\cM_{r+}^{q_1}$ is a Riemannian quotient manifold.   $\blacksquare$

§.§ Proof of Lemma <ref>

The expressions for vertical and horizontal spaces and their dimensions are provided in <cit.>. Next, we prove statement (ii). From Section <ref>, we know to show $\cM_{r+}^{q_2}$ is a Riemannian quotient manifold endowed with the Riemannian metric $g^{r+}_{[\U, \B]}$ induced from $\bar{g}^{r+}_{(\U,\B)}$, it is enough to show $\bar{g}^{r+}_{(\U,\B)} (\eta_{(\U,\B)}, \theta_{{(\U,\B)}})= \bar{g}^{r+}_{(\U\O, \O^\top \B \O )}(\eta_{(\U\O, \O^\top \B \O )}, \theta_{(\U\O, \O^\top \B \O )})$ for any $\O \in \bbO_r$, where $\eta_{(\U,\B)}, \theta_{(\U,\B)}$ and $\eta_{(\U\O, \O^\top \B \O )}, \theta_{(\U\O, \O^\top \B \O )}$ are the horizontal lifts of $\eta_{[\U, \B]}$ and $\theta_{[\U, \B]}$ at $(\U,\B)$ and $(\U\O, \O^\top \B \O )$, respectively. By Lemma <ref>(ii), we have the the horizontal lifts of $\theta_{[\U, \B]}$ at $(\U, \B)$ and $(\U \O, \O^\top \B \O)$ are related as $\theta_{(\U \O, \O^\top \B \O)} = [(\theta_U \O)^\top \quad (\O^\top \theta_{B} \O )^\top ]^\top$ given $\theta_{(\U,\B)} = [\theta_U^\top \quad \theta_B^\top]^\top$. So

\begin{equation*}
\bar{g}^{r+}_{(\U\O, \O^\top \B \O) }(\eta_{(\U\O, \O^\top \B \O) }, \theta_{(\U\O, \O^\top \B \O) }) = \tr(\V_{\O^\top \B \O} \O^\top \eta_U^\top \theta_U \O  ) +  \tr(\W_{\O^\top \B\O} \O^\top \eta_{B} \O \W_{\O^\top \B\O} \O^\top  \theta_{B} \O ), % = \tr(\W_{\Y\O} \O^\top \eta_\Y^\top \theta_\Y \O ),
\end{equation*}
and it is equal to $\bar{g}^{r+}_{(\U, \B)} (\eta_{(\U, \B)}, \theta_{(\U, \B)})$ if and only if $\V_\B = \O \V_{\O^\top \B \O} \O^\top$ and $\W_B = \O \W_{\O^\top \B \O} \O^\top$ hold for any $\O \in \bbO_r$. This finishes the proof.   $\blacksquare$

§.§ Proof of Lemma <ref>.

The results under the full-rank factorization are provided in <cit.>. Next, we prove the results under the polar factorization. If $\U_2 = \U_1 \O $ and $\B_2 = \O^\top \B_1 \O$ for some $\O \in \bbO_r$, it is easy to check $\U_1 \B_1 \U_1^\top = \U_2 \B_2 \U_2^\top$.

Now, if $\U_1 \B_1 \U_1^\top = \U_2 \B_2 \U_2^\top$ for some $\U_1, \U_2 \in \st(r,p)$, $\B_1, \B_2 \in \bbS_+(r)$. Then the column spans of $\U_1 \B_1 \U_1^\top$ and $\U_2 \B_2 \U_2^\top$ are $\U_1$ and $\U_2$, respectively and they are the same. Hence there is $\O \in \bbR^{r\times r}$ such that $\U_2 = \U_1 \O$. Moreover, since $\U_1, \U_2 \in \st(r,p)$, $\O^\top \O = \I$, i.e., $\O \in \bbO_r$. So $\U_1 \B_1 \U_1^\top = \U_2 \B_2 \U_2^\top$ and $\U_2 = \U_1 \O$ yield $\B_2 = \O^\top \B_1 \O$.

Finally, $\cM_{r+} = \{\U \B \U^\top: \U \in \st(r,p), \B \in \bbS_+(r) \}$ follows from the facts: (1) $\U \B \U^\top \in \cM_{r+}$ for $\U \in \st(r,p), \B \in \bbS_+(r)$ and (2) any matrix $\X \in \cM_{r+}$ admits eigendecomposition which has form $\U \B \U^\top$.   $\blacksquare$

§.§ Proof of Lemma <ref>.

The vertical space of $\widebar{\cM}_r^{q_1}$ was given in <cit.> and it has dimension $r^2$. Moreover, we can check $\dim(\cH_{(\L,\R)} \widebar{\cM}_{r}^{q_1}) = (p_1 + p_2-r)r$. If we can show $\cV_{(\L,\R)} \widebar{\cM}_{r}^{q_1} \perp \cH_{(\L,\R)} \widebar{\cM}_{r}^{q_1}$ with respect to $\bar{g}^{r}_{(\L,\R)}$, then it implies $\cH_{(\L,\R)} \widebar{\cM}_{r}^{q_1}$ is a valid horizontal space choice. Suppose $\theta_{(\L,\R)} = \begin{bmatrix}
\U \S \P_2^{-\top}\\
-\V \S^\top \P_1^{-\top}
\end{bmatrix} \in \cV_{(\L,\R)} \widebar{\cM}_{r}^{q_1}$ and $\eta_{(\L,\R)} = \begin{bmatrix}
(\U \S' \P_2\W_{\L, \R}^{-1} \P_2^\top + \U_\perp \D_1 ) \P_2^{-\top}\\
(\V \S^{'\top} \P_1 \V_{\L,\R}^{-1} \P_1^\top + \V_\perp \D_2  ) \P_1^{-\top}
\end{bmatrix} \in \cH_{(\L,\R)} \widebar{\cM}_{r}^{q_1}$, then by some simple calculations we have $\bar{g}_{(\L,\R)}^r ( \eta_{(\L,\R)}, \theta_{(\L,\R)} ) = \tr(\S^{'\top} \S ) - \tr(\S^{'\top} \S ) = 0$.

To show $\cM_{r}^{q_1}$ is a Riemannian quotient manifold endowed with the Riemannian metric $g^{r}_{[\L,\R]}$, it is enough to show $\bar{g}^{r}_{(\L,\R)} (\eta_{(\L,\R)}, \theta_{(\L,\R)})= \bar{g}^{r}_{(\L\M,\R\M^{-\top})}(\eta_{(\L\M,\R\M^{-\top})}, \theta_{(\L\M,\R\M^{-\top})})$ for any $\M \in \GL(r)$, where $\eta_{(\L,\R)}, \theta_{(\L,\R)}$ and $\eta_{(\L\M,\R\M^{-\top})}, \theta_{(\L\M,\R\M^{-\top})}$ are the horizontal lifts of $\eta_{[\L,\R]}$ and $\theta_{[\L,\R]}$ at $(\L,\R)$ and $(\L\M,\R\M^{-\top})$, respectively. By Lemma <ref>(iii), we have the the horizontal lifts of $\theta_{[\L,\R]}$ at $(\L,\R)$ and $(\L\M,\R\M^{-\top})$ are related as $\theta_{(\L\M,\R\M^{-\top})} = [(\theta_L \M)^\top \quad (\theta_{R} \M^{-\top})^\top ]^\top$ given $\theta_{(\L,\R)} = [\theta_L^\top \quad \theta_R^\top]^\top$. So

\begin{equation*}
\begin{split}
&\bar{g}^{r}_{(\L\M,\R\M^{-\top})}(\eta_{(\L\M,\R\M^{-\top})}, \theta_{(\L\M,\R\M^{-\top})}) \\
=& \tr\left(\W_{\L\M,\R\M^{-\top}} (\eta_L \M)^\top \theta_L \M \right) + \tr\left(\V_{\L\M,\R\M^{-\top}} (\eta_R \M^{-\top})^\top \theta_R \M^{-\top } \right),
\end{split}
\end{equation*}
and it is equal to $\bar{g}_{(\L,\R)}^r ( \eta_{(\L,\R)}, \theta_{(\L,\R)} )$ for any $\eta_{(\L,\R)}, \theta_{(\L,\R)}$ if and only if $\W_{\L,\R}$ and $\V_{\L,\R}$ satisfy the assumption given in the lemma. This shows $\cM_{r}^{q_1}$ is a Riemannian quotient manifold endowed with the Riemannian metric $g^{r}_{[\L,\R]}$. This finishes the proof.   $\blacksquare$

§.§ Proof of Lemma <ref>.

The vertical space of $\cM_{r}^{q_2}$ was given in <cit.>. Distinct from <cit.>, here the horizontal space in (<ref>) is chosen in a non-canonical way, i.e., it is not orthogonal to $\cV_{(\U,\B,\V)} \widebar{\cM}_{r}^{q_2}$ with respect to $\bar{g}_{(\U,\B,\V)}^r$, so we need to verify $\cH_{(\U,\B,\V)} \widebar{\cM}_{r}^{q_2}$ in (<ref>) is a valid horizontal space choice. To show this, we need to show $T_{(\U,\B,\V)} \widebar{\cM}_r^{q_2} = \cV_{(\U,\B,\V)} \widebar{\cM}_{r}^{q_2} \oplus \cH_{(\U,\B,\V)} \widebar{\cM}_{r}^{q_2}$, where $T_{(\U,\B,\V)} \widebar{\cM}_r^{q_2} = T_\U\st(r,p_1) \times T_\B \bbS_+(r) \times T_\V \st(r,p_2)$.

First, it is easy to check $\dim(\cV_{(\U,\B,\V)} \widebar{\cM}_{r}^{q_2}) = (r^2-r)/2$, $\dim(\cH_{(\U,\B,\V)} \widebar{\cM}_{r}^{q_2}) = (p_1 + p_2 - r)r$ and $\dim( T_{(\U,\B,\V)} \widebar{\cM}_r^{q_2} ) = (p_1 + p_2 - r)r + (r^2-r)/2$. So $\dim(\cV_{(\U,\B,\V)} \widebar{\cM}_{r}^{q_2}) + \dim(\cH_{(\U,\B,\V)} \widebar{\cM}_{r}^{q_2}) = \dim( T_{(\U,\B,\V)} \widebar{\cM}_r^{q_2} )$. Moreover, for $\eta_{(\U,\B,\V)} \in \cV_{(\U,\B,\V)} \widebar{\cM}_{r}^{q_2}$, $\theta_{(\U,\B,\V)} \in \cH_{(\U,\B,\V)} \widebar{\cM}_{r}^{q_2}$, we have $\eta_{(\U,\B,\V)} + \theta_{(\U,\B,\V)} \in T_{(\U,\B,\V)} \widebar{\cM}_r^{q_2}$. Finally, given $\xi_{(\U,\B, \V)} = [(\U \bOmega + \U_\perp \D)^\top \quad \S \quad (\V \bOmega' + \V_\perp \D')^\top ]^\top$, we can find $\eta_{(\U,\B,\V)} = [(\U \bOmega_1)^\top \quad (\B\bOmega_1 - \bOmega_1\B)^\top \quad (\V \bOmega_1)^\top  ]^\top \in \cV_{(\U,\B,\V)} \widebar{\cM}_{r}^{q_2}$, $\theta_{(\U,\B,\V)} = [ (\U_\perp \D_1 + \U \bOmega_2 )^\top \quad \theta_B \quad (\V_\perp \D_2 - \V \bOmega_2 )^\top ]^\top \in \cH_{(\U,\B,\V)} \widebar{\cM}_{r}^{q_2}$ such that $\eta_{(\U,\B,\V)} + \theta_{(\U,\B,\V)} = \xi_{(\U,\B, \V)}$, where $\bOmega_1, \bOmega_2,\theta_B,\D_1,\D_2$ are uniquely determined by the equation system $\bOmega_1 + \bOmega_2 = \bOmega, \bOmega_1 - \bOmega_2 = \bOmega',\theta_B = \S - (\B \bOmega_1 - \bOmega_1 \B), \D_1 = \D, \D_2 = \D'.$ This proves $T_{(\U,\B,\V)} \widebar{\cM}_r^{q_2} = \cV_{(\U,\B,\V)} \widebar{\cM}_{r}^{q_2} \oplus \cH_{(\U,\B,\V)} \widebar{\cM}_{r}^{q_2}$

The claim $\cM_{r}^{q_2}$ endowed with metric $g^{r}_{[\U,\B,\V]}$ induced from $\bar{g}^{r}_{(\U,\B,\V)}$ is a Riemannian quotient manifold has appeared in <cit.> without a proof, here we provide the proof for completeness. The result follows if we can show
$$\bar{g}^{r}_{(\U,\B,\V)} (\eta_{(\U,\B,\V)}, \theta_{(\U,\B,\V)})= \bar{g}^{r}_{(\U\O,\O^\top\B\O,\V\O)}(\eta_{(\U\O,\O^\top\B\O,\V\O)}, \theta_{(\U\O,\O^\top\B\O,\V\O)})$$
holds for any $\O \in \bbO_r$, where $\eta_{(\U,\B,\V)}, \theta_{(\U,\B,\V)}$ and $\eta_{(\U\O,\O^\top\B\O,\V\O)}, \theta_{(\U\O,\O^\top\B\O,\V\O)}$ are the horizontal lifts of $\eta_{[\U,\B,\V]}$ and $\theta_{[\U,\B,\V]}$ at $(\U,\B,\V)$ and $(\U\O,\O^\top\B\O,\V\O)$, respectively. By Lemma <ref>(iv), we have the the horizontal lifts of $\theta_{[\U,\B,\V]}$ at $(\U,\B,\V)$ and $(\U\O,\O^\top\B\O,\V\O)$ are related as $\theta_{(\U\O,\O^\top\B\O,\V\O)} = [(\theta_U \O)^\top \quad (\O^\top \theta_{B} \O)^\top \quad (\theta_V \O)^\top ]^\top$ given $\theta_{(\U,\B,\V)} = [\theta_U^\top \quad \theta_B^\top \quad \theta_V^\top]^\top$. So

\begin{equation*}
\begin{split}
&\bar{g}^{r}_{(\U\O,\O^\top\B\O,\V\O)}(\eta_{(\U\O,\O^\top\B\O,\V\O)}, \theta_{(\U\O,\O^\top\B\O,\V\O)})\\
= & \tr(\O^\top \eta_U^\top \theta_U \O) + \tr\left((\O^\top\B\O)^{-1} \O^\top \eta_B \O (\O^\top\B\O)^{-1} \O^\top \theta_B \O   \right) + \tr(\O^\top \eta_V^\top \theta_V \O)\\
= & \tr(\eta_U^\top \theta_U ) + \tr(\B^{-1} \eta_B \B^{-1} \theta_B ) + \tr(\eta_V^\top \theta_V) = \bar{g}_{(\U,\B,\V)}^r ( \eta_{(\U,\B,\V)}, \theta_{(\U,\B,\V)} ).
\end{split}
\end{equation*}
This shows $\cM_{r}^{q_2}$ endowed with the metric $g^{r}_{[\U,\B,\V]}$ is a Riemannian quotient manifold. This finishes the proof of this lemma.   $\blacksquare$

§.§ Proof of Lemma <ref>.

The vertical space of $\cM_r^{q_3}$ was given in <cit.>. Distinct from <cit.>, here the horizontal space in (<ref>) is chosen in a non-canonical way, so we need to verify $\cH_{(\U,\Y)} \widebar{\cM}_{r}^{q_3}$ in (<ref>) is a valid horizontal space choice. To show this, we need to show $T_{(\U,\Y)} \widebar{\cM}_r^{q_3} = \cV_{(\U,\Y)} \widebar{\cM}_{r}^{q_3} \oplus \cH_{(\U,\Y)} \widebar{\cM}_{r}^{q_3}$, where $T_{(\U,\Y)} \widebar{\cM}_r^{q_3} = T_\U\st(r,p_1) \times \bbR^{p_2 \times r}_*$.

First, it is easy to check $\dim(\cV_{(\U,\Y)} \widebar{\cM}_{r}^{q_3}) = (r^2-r)/2$, $\dim(\cH_{(\U,\Y)} \widebar{\cM}_{r}^{q_3}) = (p_1 + p_2 - r)r$ and $\dim( T_{(\U,\Y)} \widebar{\cM}_r^{q_3} ) = (p_1 + p_2 - r)r + (r^2-r)/2$. So $\dim(\cV_{(\U,\Y)} \widebar{\cM}_{r}^{q_3}) + \dim(\cH_{(\U,\Y)} \widebar{\cM}_{r}^{q_3}) = \dim( T_{(\U,\Y)} \widebar{\cM}_r^{q_3} )$. Moreover, for $\eta_{(\U,\Y)} \in \cV_{(\U,\Y)} \widebar{\cM}_{r}^{q_3}$, $\theta_{(\U,\Y)} \in \cH_{(\U,\Y)} \widebar{\cM}_{r}^{q_3}$, we have $\eta_{(\U,\Y)} + \theta_{(\U,\Y)} \in T_{(\U,\Y)} \widebar{\cM}_r^{q_3}$. Finally, given $\xi_{(\U,\Y)} = [(\U \bOmega + \U_\perp \D)^\top \quad \xi_Y^\top ]^\top$, we can find $\eta_{(\U,\Y)} = [(\U \bOmega_1)^\top \quad (\Y \bOmega_1)^\top  ]^\top \in \cV_{(\U,\Y)} \widebar{\cM}_{r}^{q_3}$, $\theta_{(\U,\Y)} = [ (\U_\perp \D_1)^\top \quad \theta_Y^\top ]^\top \in \cH_{(\U,\Y)} \widebar{\cM}_{r}^{q_3}$ such that $\eta_{(\U,\Y)} + \theta_{(\U,\Y)} = \xi_{(\U,\Y)}$, where $\bOmega_1, \D_1, \theta_Y$ are uniquely determined as by the equation system $\bOmega_1 = \bOmega, \D_1 = \D, \theta_Y = \xi_Y - \Y \bOmega_1.$ This proves $T_{(\U,\Y)} \widebar{\cM}_r^{q_3} = \cV_{(\U,\Y)} \widebar{\cM}_{r}^{q_3} \oplus \cH_{(\U,\Y)} \widebar{\cM}_{r}^{q_3}$.

To show $\cM_{r}^{q_3}$ is a Riemannian quotient manifold, we only need to show $\bar{g}^{r}_{(\U, \Y)} (\eta_{(\U, \Y)}, \theta_{(\U,\Y)})= \bar{g}^{r}_{(\U\O,\Y\O)}(\eta_{(\U\O,\Y\O)}, \theta_{(\U\O,\Y\O)})$ for any $\O \in \bbO_r$, where $\eta_{(\U, \Y)}, \theta_{(\U, \Y)}$ and $\eta_{(\U\O,\Y\O)}, \theta_{(\U\O,\Y\O)}$ are the horizontal lifts of $\eta_{[\U,\Y]}$ and $\theta_{[\U,\Y]}$ at $(\U,\Y)$ and $(\U\O,\Y\O)$, respectively. By Lemma <ref>(v), we have the the horizontal lifts of $\eta_{[\U,\Y]}$ at $(\U,\Y)$ and $(\U\O,\Y\O)$ are related as $\eta_{(\U\O,\Y\O)} = [(\eta_U \O)^\top \quad (\eta_Y \O)^\top]^\top$ given $\eta_{(\U, \Y)} = [\eta_U^\top \quad \eta_Y^\top]^\top$. So

\begin{equation*}
\begin{split}
\bar{g}^{r}_{(\U\O,\Y\O)}(\eta_{(\U\O,\Y\O)}, \theta_{(\U\O,\Y\O)}) = \tr( \V_{\Y \O} \O^\top \eta_U^\top \theta_U \O) + \tr( \W_{\Y \O} \O^\top \eta_Y^\top \theta_Y \O ),
\end{split}
\end{equation*}
and it is equal to $\bar{g}^{r}_{(\U, \Y)} (\eta_{(\U, \Y)}, \theta_{(\U,\Y)})$ for any $\eta_{(\U, \Y)}, \theta_{(\U,\Y)}$ if and only if $\V_\Y = \O \V_{\Y\O} \O^\top$ and $\W_\Y = \O \W_{\Y\O} \O^\top$ holds for any $\O \in \bbO_r$. This shows $\cM_{r}^{q_3}$ is a Riemannian quotient manifold endowed with metric $g_{[\U,\Y]}^r$ induced from $\bar{g}_{(\U,\Y)}^r$. This finishes the proof of this lemma.   $\blacksquare$

§.§ Proof of Lemma <ref>.

(Full-rank factorization) If $\L_2 = \L_1 \M$ and $\R_2 = \R_1 \M^{-\top} $ for some $\M \in \GL(r)$, it is easy to check $\L_1 \R_1^\top = \L_2 \R_2^\top$.

Now, if $\L_1 \R_1^\top = \L_2 \R_2^\top$ for some $\L_1, \L_2 \in \bbR_*^{p_1 \times r}$ and $\R_1, \R_2 \in \bbR_*^{p_2 \times r}$. Then the column spans of $\L_1 \R_1^\top$ and $\L_2 \R_2^\top$ are $\rspan(\L_1)$ and $\rspan(\L_2)$, respectively and they are the same. Hence there exists $\M \in \GL(r)$ such that $\L_2 = \L_1 \M$. Thus, $\L_1 \R_1^\top = \L_2 \R_2^\top$ yields $\R_2 = \R_1 \M^{-\top}$.

Moreover, $\cM_r = \{\L \R^\top: \L \in \bbR^{p_1 \times r}_*, \R \in \bbR_*^{p_2 \times r} \}$ follows from the facts: (1) $\L \R^\top \in \cM_r$ for any $\L \in \bbR^{p_1 \times r}_*, \R \in \bbR_*^{p_2 \times r}$ and (2) any matrix $\X \in \cM_r$ admits SVD $\U \bSigma \V^\top$ and by taking $\L = \U \bSigma^{1/2}$ and $\R = \V \bSigma^{1/2} $, we have $\X = \L \R^\top$.

(Polar factorization) If $\U_2 = \U_1 \O $, $\B_2 = \O^\top \B_1 \O$ and $\V_2 = \V_1 \O$ for some $\O \in \bbO_r$, it is easy to check $\U_1 \B_1 \V_1^\top = \U_2 \B_2 \V_2^\top$.

Now, if $\U_1 \B_1 \V_1^\top = \U_2 \B_2 \V_2^\top$ for some $\U_1, \U_2 \in \st(r,p_1)$, $\B_1, \B_2 \in \bbS_+(r)$ and $\V_1, \V_2 \in \st(r,p_2)$. Then the column spans of $\U_1 \B_1 \V_1^\top$ and $\U_2 \B_2 \V_2^\top$ are $\rspan(\U_1)$ and $\rspan(\U_2)$, respectively and they are the same. Hence there exists $\O_1 \in \bbR^{r\times r}$ such that $\U_2 = \U_1 \O_1$. Moreover, since $\U_1, \U_2 \in \st(r,p)$, $\O_1^\top \O_1 = \I$, i.e., $\O_1 \in \bbO_r$. Similarly, we can conclude $\V_2 = \V_1 \O_2$ for some $\O_2 \in \bbO_r$. Thus, $\U_1 \B_1 \V_1^\top = \U_2 \B_2 \V_2^\top$ yields $\B_2 = \O_1^\top \B_1 \O_2$. Since $\B_2 \in \bbS_+(r)$, by the characterization of PSD factorization in Lemma <ref>, we know $\O_1 = \O_2$.

Moreover, $\cM_r =\{\U \B \V^\top: \U \in \st(r,p_1), \B \in \bbS_+(r), \V \in \st(r, p_2) \}$ follows from the facts: (1) $\U \B \V^\top \in \cM_r$ for any $\U \in \st(r,p_1), \B \in \bbS_+(r), \V \in \st(r, p_2)$ and (2) any matrix $\X \in \cM_r$ admits SVD which has form $\U \B \V^\top$.

(Subspace-projection factorization) If $\U_2 = \U_1 \O $ and $\Y_2 = \Y_1 \O$ for some $\O \in \bbO_r$, it is easy to check $\U_1 \Y_1^\top = \U_2 \Y_2^\top$.

Now, if $\U_1 \Y_1^\top = \U_2 \Y_2^\top$ for some $\U_1, \U_2 \in \st(r,p_1)$ and $\Y_1, \Y_2 \in \bbR_*^{p_2 \times r}$. Then the column spans of $\U_1 \Y_1^\top$ and $\U_2 \Y_2^\top$ are $\rspan(\U_1)$ and $\rspan(\U_2)$, respectively and they are the same. Following the same argument as in the Polar factorization, we know $\U_2 = \U_1 \O$ for some $\O \in \bbO_r$. Thus, $\U_1 \Y_1^\top = \U_2 \Y_2^\top$ yields $\Y_2 = \Y_1 \O$.

Moreover, $\cM_r = \{\U \Y^\top: \U \in \st(r,p_1), \Y \in \bbR_*^{p_2 \times r} \}$ follows from the facts: (1) $\U \Y^\top \in \cM_r$ for any $ \U \in \st(r,p_1), \Y \in \bbR_*^{p_2 \times r} $ and (2) any matrix $\X \in \cM_{r}$ admits SVD $\U \bSigma \V^\top$ and by taking $\Y = \V \bSigma$, we have $\X = \U \Y^\top$.   $\blacksquare$

§ ADDITIONAL PROOFS IN SECTION <REF>

§.§ Proof of Proposition <ref>.

The proof is divided into four steps: in Step 1, we derive the expressions for Riemannian gradients; in Step 2, we derive the Riemannian Hessian of (<ref>) under the embedded geometry; in Step 3, we derive the Riemannian Hessian for $h_{r+}([\Y])$; in Step 4, we derive the Riemannian Hessian for $h_{r+}([\U,\B])$.

Step 1. First, the Riemannian gradient expression of (<ref>) under the embedded geometry can be found in <cit.>. Next, we compute the Riemannian gradients under the quotient geometry. By definition

\begin{equation} \label{eq: gradient-comp-psd-1}
\begin{split}
\langle \grad \, \bar{h}_{r+}(\Y) \W_\Y, \eta_\Y \rangle=\bar{g}_\Y^{r+}(\grad \, \bar{h}_{r+}(\Y), \eta_\Y)  &= \rmD \bar{h}_{r+}(\Y)[\eta_\Y] = \langle \nabla \bar{h}_{r+}(\Y), \eta_\Y \rangle \\
& = 2\langle \nabla f(\Y \Y^\top) \Y, \eta_\Y \rangle, \quad \forall \eta_\Y \in T_\Y \widebar{\cM}_{r+}^{q_1},
\end{split}
\end{equation}
where $\nabla \bar{h}_{r+}(\Y)$ denotes the Euclidean gradient of $\bar{h}_{r+}$ at $\Y$, and

\begin{equation} \label{eq: gradient-comp-psd-2}
\begin{split}
&\langle  \grad_\U\, \bar{h}_{r+}(\U, \B)\V_\B, \eta_U \rangle + \langle \W_\B \grad_\B\, \bar{h}_{r+}(\U, \B) \W_\B, \eta_B \rangle \\
=& \bar{g}^{r+}_{(\U, \B)} ( \grad\, \bar{h}_{r+}(\U, \B), \eta_{(\U, \B)} ) \\
=& \rmD \bar{h}_{r+}(\U, \B)[\eta_{(\U, \B)}] = \langle \nabla_\U \bar{h}_{r+}(\U,\B) , \eta_U \rangle + \langle \nabla_\B \bar{h}_{r+}(\U,\B) , \eta_B \rangle \\
=& 2\langle \nabla f(\U \B \U^\top)\U \B, \eta_U \rangle + \langle \U^\top \nabla f(\U \B \U^\top) \U, \eta_B \rangle, \quad \forall \eta_{(\U, \B)} \in T_{(\U, \B)} \widebar{\cM}_{r+}^{q_2}.
\end{split}
\end{equation}
From (<ref>), we have the following expression for $\grad \, \bar{h}_{r+}(\Y)$:

\begin{equation} \label{eq: Rgradient-total-space-psd-quotient}
\begin{split}
\grad \, \bar{h}_{r+}(\Y) = \nabla \bar{h}_{r+}(\Y) \W_\Y^{-1} = 2\nabla f(\Y \Y^\top) \Y \W_\Y^{-1}.
\end{split}
\end{equation}
Since $\cV_\Y \widebar{\cM}_{r+}^{q_1} \perp \cH_\Y \widebar{\cM}_{r+}^{q_1}$, by Lemma <ref>, we have $\overline{\grad \, h_{r+}([\Y])} = \grad \, \bar{h}_{r+}(\Y)$.

At the same time, from (<ref>), we have $\grad_\B \, \bar{h}_{r+}(\U, \B) =\W^{-1}_\B \U^\top \nabla f(\U \B \U^\top) \U \W^{-1}_\B $. In addition, suppose $\grad_\U \, \bar{h}_{r+}(\U, \B) = \U_\perp \D + \U \bOmega \in T_\U \st(r,p)$, then by (<ref>), we have $P_{ T_\U \st(r,p) } ( \U_\perp \D \V_\B + \U \bOmega \V_\B - 2 \nabla f(\U \B \U^\top)\U \B ) = 0$. Thus $\D = 2 \U_\perp^\top \nabla f(\U \B \U^\top) \U \B \V_\B^{-1} $ and $\bOmega$ is uniquely determined by the following Sylvester equation $\V_\B \bOmega^\top + \bOmega^\top \V_\B = 2( \B \U^\top \nabla f(\U \B \U^\top) \U - \U^\top \nabla f( \U \B \U^\top) \U \B  ) $ . So

\begin{equation*}
\overline{\grad \, h_{r+}([\U, \B])} = P_{(\U, \B)}^{\cH} (\grad \, \bar{h}_{r+}(\U, \B)) = \begin{bmatrix}
2 P_{\U_\perp} \nabla f(\U \B \U^\top) \U \B \V_\B^{-1} \\
\W_\B^{-1} \U^\top \nabla f(\U \B \U^\top) \U \W_\B^{-1}
\end{bmatrix},
\end{equation*}
here $P_{(\U, \B)}^{\cH}(\cdot)$ is the projection operator onto $\cH_{(\U,\B)} \widebar{\cM}^{q_1}_{r+}$ and it satisfies $P_{(\U, \B)}^{\cH}(\eta_{(\U,\B)}) = [(P_{\U_\perp} \eta_U)^\top \quad \eta_B^\top ]^\top$ for any $\eta_{(\U,\B)} \in T_{(\U,\B)} \widebar{\cM}_{r+}^{q_2}$.

Step 2.  Suppose $\X$ has eigendecomposition $\U' \bSigma' \U^{'\top}$ with $\U = \U' \O$ and $\O \in \bbO_r$. Then for

\begin{equation*}
\begin{split}
\xi_\X &= [\U \quad \U_\perp] \begin{bmatrix}
\S & \D^\top\\
\D & \0
\end{bmatrix} [\U \quad \U_\perp]^\top = [\U' \quad \U_\perp] \begin{bmatrix}
\O\S \O^\top & \O\D^\top\\
\D\O^\top & \0
\end{bmatrix} [\U' \quad \U_\perp]^\top,
\end{split}
\end{equation*}
by <cit.> we have in the PSD case

\begin{equation*}
\begin{split}
\Hess f(\X)[\xi_\X, \xi_\X] &= \nabla^2 f(\X)[\xi_\X, \xi_\X] + 2\langle \nabla f(\X), \U_\perp \D \O^\top \bSigma^{'-1} \O \D^\top \U_\perp^\top \rangle \\
& = \nabla^2 f(\X)[\xi_\X, \xi_\X] + 2\langle \nabla f(\X), \U_\perp \D \bSigma^{-1} \D^\top \U_\perp^\top \rangle
\end{split}
\end{equation*}
where $\bSigma = \O^\top \bSigma^{'} \O = \U^\top \X \U$.

Step 3. We first compute

\begin{equation} \label{eq: directional-derivative-quotient-gradient-psd-1}
\begin{split}
\rmD \overline{ \grad \, h_{r+}([\Y])}[\theta_\Y] = \rmD \grad \, \bar{h}_{r+}(\Y)[\theta_\Y] & \overset{ \eqref{eq: Rgradient-total-space-psd-quotient} } =  \lim_{t \to 0} \left( \nabla \bar{h}_{r+} (\Y + t \theta_\Y) \W_{\Y + t\theta_\Y}^{-1} - \nabla \bar{h}_{r+}(\Y) \W_\Y^{-1}  \right)/t\\
& = \nabla \bar{h}_{r+}(\Y) \rmD \W_\Y^{-1} [\theta_\Y] + \nabla^2 \bar{h}_{r+}(\Y)[\theta_\Y] \W_\Y^{-1}.
\end{split}
\end{equation}

By the definition of the bilinear form of the Riemannian Hessian, we have

\begin{equation} \label{eq: quotient-hessian-psd-manifold1}
\begin{split}
\overline{\Hess \, h_{r+}([\Y])}[\theta_\Y, \theta_\Y] &= \bar{g}_\Y^{r+} (\overline{\Hess \, h_{r+}([\Y])[\theta_{[\Y]}]}, \theta_\Y )\\
&\overset{ \eqref{eq: quotient-hessian-linear-from} } = \bar{g}_\Y^{r+} \left( P_{\Y}^{\cH} \left( \widebar{\nabla}_{\theta_\Y} \overline{ \grad \, h_{r+}([\Y]) } \right) , \theta_\Y \right) \\
& \overset{ \text{Lemma } \ref{lm: quotient-hessian-riemannian-connection-equal} } = \bar{g}_\Y^{r+} \left( \widebar{\nabla}_{\theta_\Y} \overline{ \grad \, h_{r+}([\Y])} , \theta_\Y \right)\\
& \overset{ \eqref{eq: koszul-formula-simple-manifolds} } = \bar{g}_\Y^{r+} \left( \rmD \overline{ \grad \, h_{r+}([\Y])}[\theta_\Y] , \theta_\Y \right) + \tr\left( \rmD \W_\Y\left[\overline{ \grad \, h_{r+}([\Y])}\right] \theta_\Y^\top \theta_\Y  \right)/2\\
& \overset{ \eqref{eq: Rgradient-total-space-psd-quotient}, \eqref{eq: directional-derivative-quotient-gradient-psd-1} } = \nabla^2 \bar{h}_{r+}(\Y)[\theta_\Y, \theta_\Y] + 2\tr\left(\W_\Y \left( \nabla f(\Y \Y^\top) \Y  \rmD \W_\Y^{-1} [\theta_\Y]\right)^\top \theta_\Y \right) \\
& \quad + \tr\left( \rmD \W_\Y\left[\overline{ \grad \, h_{r+}([\Y])}\right] \theta_\Y^\top \theta_\Y  \right)/2.
\end{split}
\end{equation}
Finally, we have $\nabla^2 \bar{h}_{r+}(\Y)[\theta_\Y, \theta_\Y] = \nabla^2 f(\Y \Y^\top)[\Y\theta_\Y^\top + \theta_\Y \Y^\top , \Y\theta_\Y^\top + \theta_\Y \Y^\top ] + 2\langle \nabla f(\Y \Y^\top ), \theta_\Y \theta_\Y^\top \rangle$ from <cit.> and plugging it into (<ref>), we get the expression for the Riemannian Hessian of $h_{r+}([\Y])$.

Step 4.
For convenience, we denote $\bar{g}_{\U}^{r+}(\eta_U, \theta_U) = \tr(\V_\B \eta_U^\top \theta_U) $ as the $\U$ part inner product of $\bar{g}^{r+}_{(\U, \B)}$ and $\bar{g}_{\B}^{r+}(\eta_B, \theta_B) = \tr(\W_\B \eta_B \W_\B \theta_B)$ as the $\B$ part inner product of $\bar{g}^{r+}_{(\U, \B)}$. We have

\begin{equation} \label{eq: quotient-hessian-manifold2}
\begin{split}
& \quad \overline{\Hess \, h_{r+}([\U,\B])}[\theta_{(\U,\B)}, \theta_{(\U,\B)}] \\
&= \bar{g}_{(\U, \B)}^{r+}\left( \overline{\Hess \, h_{r+}([\U,\B])[\theta_{[\U,\B]}]} , \theta_{(\U,\B)} \right)\\
&\overset{ \eqref{eq: quotient-hessian-linear-from} } = \bar{g}_{(\U, \B)}^{r+} \left( P_{(\U, \B)}^{\cH} \left( \widebar{\nabla}_{\theta_{(\U, \B)}} \overline{ \grad \, h_{r+}([\U, \B]) } \right) , \theta_{(\U, \B)} \right)\\
& \overset{ \text{Lemma } \ref{lm: quotient-hessian-riemannian-connection-equal} } = \bar{g}_{(\U, \B)}^{r+} \left( \widebar{\nabla}_{\theta_{(\U, \B)}} \overline{ \grad \, h_{r+}([\U, \B])} , \theta_{(\U, \B)} \right)\\
& \overset{ \eqref{eq: koszul-formula-st-s+} } = \bar{g}_{\U}^{r+} \left( \rmD \overline{ \grad_\U \, h_{r+}([\U, \B])}[\theta_{(\U, \B)}] , \theta_{U} \right) + \bar{g}_{\B}^{r+} \left( \rmD \overline{ \grad_\B \, h_{r+}([\U, \B])}[\theta_{(\U, \B)}], \theta_B \right)\\
&\quad  + \tr\left( \rmD \V_\B [ \overline{ \grad_\B \, h_{r+}([\U, \B])} ] \theta_U^\top \theta_U \right)/2\\
&\quad   + \tr( \sym( \W_\B \theta_B \rmD \W_\B [\overline{ \grad_\B \, h_{r+}([\U, \B])}  ] ) \theta_B   ).
\end{split}
\end{equation}
Next, we compute $\rmD \overline{ \grad_\U \, h_{r+}([\U, \B])}[\theta_{(\U, \B)}]$ and $\rmD \overline{ \grad_\B \, h_{r+}([\U, \B])}[\theta_{(\U, \B)}]$ separately in (<ref>).

\begin{equation} \label{eq: quotient-hessian-manifold2-comp1}
\begin{split}
&\rmD \overline{ \grad_\U \, h_{r+}([\U, \B])}[\theta_{(\U, \B)}]\\
= & 2 \lim_{t \to 0} \Big( (\I_p - P_{\U + t \theta_U}) \nabla f\left( (\U + t\theta_U ) (\B+t\theta_B) (\U + t\theta_U)^\top\right) (\U + t \theta_U) (\B + t\theta_B)\V^{-1}_{\B+ t \theta_B} \\
& \quad \quad - P_{\U_\perp} \nabla f(\U \B \U^\top) \U \B \V^{-1}_{\B} \Big)/t\\
=&  -4 \sym(\U \theta_U^\top) \nabla f(\U \B \U^\top) \U \B\V^{-1}_{\B} + 2 P_{\U_\perp}\left( \nabla f(\U \B \U^\top) \theta_U \B\V^{-1}_{\B} + \nabla f(\U \B \U^\top) \U \theta_B \V^{-1}_{\B} \right) \\
& + 2 P_{\U_\perp} \nabla f(\U \B \U^\top) \U \B \rmD \V_{\B}^{-1}[\theta_B]  \\
& +   2 \lim_{t \to 0} P_{\U_\perp}\left(  \nabla f\left( (\U + t\theta_U ) (\B+t\theta_B) (\U + t\theta_U)^\top\right) - \nabla f(\U \B \U^\top) \right) \U \B\V^{-1}_{\B}/t\\
=& -4 \sym(\U \theta_U^\top) \nabla f(\U \B \U^\top) \U \B\V^{-1}_{\B} + 2 P_{\U_\perp}\left( \nabla f(\U \B \U^\top) \theta_U \B\V^{-1}_{\B} + \nabla f(\U \B \U^\top) \U \theta_B \V^{-1}_{\B} \right) \\
& + 2 P_{\U_\perp} \nabla f(\U \B \U^\top) \U \B \rmD \V_{\B}^{-1}[\theta_B]  \\
&+ 2 P_{\U_\perp} \nabla^2 f(\U\B\U^\top)[ \U \B \theta_U^\top+ \U \theta_B \U^\top  + \theta_U \B \U^\top] \U \B\V^{-1}_{\B},
\end{split}
\end{equation}

\begin{equation} \label{eq: quotient-hessian-manifold2-comp2}
\begin{split}
&\rmD \overline{ \grad_\B \, h_{r+}([\U, \B])}[\theta_{(\U, \B)}]\\
=& \lim_{t \to 0} \big( \W^{-1}_{\B + t \theta_B} (\U + t\theta_U)^\top \nabla f\left((\U + t\theta_U) (\B + t\theta_B) (\U + t\theta_U)^\top\right) (\U + t\theta_U) \W^{-1}_{\B + t \theta_B} \\
& - \W^{-1}_{\B} \U^\top \nabla f(\U \B \U^\top) \U \W^{-1}_{\B} \big)/t\\
=& (\rmD \W_\B^{-1}[\theta_B] \U^\top + \W^{-1}_{\B}  \theta_U^\top) \nabla f(\U \B \U^\top) \U \W^{-1}_{\B}   + \W^{-1}_{\B} \U^\top \nabla f(\U \B \U^\top) (\U\rmD \W_\B^{-1}[\theta_B]  + \theta_U \W^{-1}_{\B} ) \\
& + \W^{-1}_{\B}  \U^\top  \nabla^2 f(\U\B\U^\top)[ \U \B \theta_U^\top+ \U \theta_B \U^\top  + \theta_U \B \U^\top] \U \W^{-1}_{\B} .
\end{split}
\end{equation}

By observing $f(\X)$ is symmetric in $\X$, $\theta_{(\U, \B)} \in \cH_{(\U, \B)} \widebar{\cM}_{r+}^{q_2}$ and plugging (<ref>) and (<ref>) into (<ref>), we have

\begin{equation*}
\begin{split}
&\overline{\Hess \, h_{r+}([\U,\B])}[\theta_{(\U,\B)}, \theta_{(\U,\B)}] \\
=& \nabla^2 f(\U\B\U^\top)[ \U \B \theta_U^\top+ \U \theta_B \U^\top  + \theta_U \B \U^\top,  \U \B \theta_U^\top + \theta_U \B \U^\top] + 2 \langle  \nabla f(\U \B \U^\top), \theta_U \B \theta_U^\top \rangle \\
& - 4\langle \sym(\U \theta_U^\top) \nabla f(\U \B \U^\top) \U \B, \theta_U \rangle + 2 \langle \nabla f(\U \B \U^\top) \U \theta_B , \theta_U \rangle \\
& + 2 \langle \nabla f(\U \B \U^\top) \U \B \rmD \V_\B^{-1} [\theta_B] \V_\B, \theta_U \rangle \\
& + \nabla^2 f(\U\B\U^\top)[ \U \B \theta_U^\top+ \U \theta_B \U^\top  + \theta_U \B \U^\top,  \U \theta_B \U^\top]\\
&  + \tr(\W_{\B} (\rmD \W_\B^{-1} [\theta_B] \U^\top + \W^{-1}_{\B} \theta_U^\top) \nabla f(\U \B \U^\top) \U  \theta_B ) \\
& + \tr(\U^\top \nabla f(\U \B \U^\top) (\U \rmD \W_{\B}^{-1} [\theta_B]  + \theta_U \W^{-1}_{\B})  \W_{\B} \theta_B )\\
& + \tr\left( \rmD \V_\B [ \overline{ \grad_\B \, h_{r+}([\U, \B])} ] \theta_U^\top \theta_U \right)/2 \\
& + \tr( \sym( \W_\B \theta_B \rmD \W_\B [\overline{ \grad_\B \, h_{r+}([\U, \B])}  ] ) \theta_B   ) \\
=& \nabla^2 f(\U \B \U^\top)[\U \B \theta_U^\top + \U \theta_B \U^\top + \theta_U \B \U^\top, \U \B \theta_U^\top + \U \theta_B \U^\top + \theta_U \B \U^\top]  + 2 \langle  \nabla f(\U \B \U^\top), \theta_U \B \theta_U^\top \rangle\\
& \,+2\langle \nabla f(\U \B \U^\top) \U, 2 \theta_U \theta_B + \U \rmD \W_\B^{-1}[ \theta_B] \W_\B \theta_B + \theta_U \V_\B \rmD \V_\B^{-1}[\theta_B] \B - \theta_U \U^\top \theta_U \B - \U \theta_U^\top \theta_U \B \rangle \\
& \, + \tr( \rmD \V_\B [ \overline{ \grad_\B \, h_{r+}([\U, \B])}  ] \theta_U^\top \theta_U   )/2 + \tr( \sym(  \W_\B \theta_B \rmD \W_\B [  \overline{ \grad_\B \, h_{r+}([\U, \B])}  ]  ) \theta_B ). \quad \blacksquare
\end{split}
\end{equation*}

§.§ Proof of Theorem <ref>.

First, (<ref>) is by direct calculation from the gradient expressions in Proposition <ref>. Next, we prove (<ref>). Since $[\U,\B]$ is a Riemannian FOSP of (<ref>), we have

\begin{equation} \label{eq: FOSP-condition-PSD2}
\overline{ \grad \, h_{r+}([\U,\B])} = \0 \quad \text{ and } \quad \nabla f(\U \B \U^\top) \U = \0
\end{equation}
and have $\grad f(\X) = \0$ by (<ref>). So $ \nabla f(\X) = P_{\U_\perp} \nabla f(\X) P_{\U_\perp}$. Given any $\theta_{(\U,\B)} = [\theta_U^\top \quad \theta_B^\top]^\top \in \cH_{(\U,\B)} \widebar{\cM}_{r+}^{q_2}$, we have

\begin{equation} \label{eq: Hessian-con-gradient-2}
\langle \nabla f(\X), P_{\U_\perp} \theta_U \B \B^{-1} \B \theta_U^\top  P_{\U_\perp}    \rangle = \langle \nabla f(\X), \theta_U \B \theta_U^\top \rangle,
\end{equation}
where the equality is because $ \nabla f(\X) = P_{\U_\perp} \nabla f(\X) P_{\U_\perp} $.

Then by Proposition <ref>:

\begin{equation*}%\label{eq: Hessian-con-Hessian-2}
\begin{split}
& \quad \overline{\Hess \, h_{r+}([\U,\B])}[\theta_{(\U,\B)}, \theta_{(\U,\B)}] \\
&= \nabla^2 f(\U \B \U^\top)[\U \B \theta_U^\top + \U \theta_B \U^\top + \theta_U \B \U^\top, \U \B \theta_U^\top + \U \theta_B \U^\top + \theta_U \B \U^\top]  + 2 \langle  \nabla f(\U \B \U^\top), \theta_U \B \theta_U^\top \rangle\\
& \,+2\langle \nabla f(\U \B \U^\top) \U, 2 \theta_U \theta_B + \U \rmD \W_\B^{-1}[ \theta_B] \W_\B \theta_B + \theta_U \V_\B \rmD \V_\B^{-1}[\theta_B] \B - \theta_U \U^\top \theta_U \B - \U \theta_U^\top \theta_U \B \rangle \\
& \, + \tr( \rmD \V_\B [ \overline{ \grad_\B \, h_{r+}([\U, \B])}  ] \theta_U^\top \theta_U   )/2 + \tr( \sym(  \W_\B \theta_B \rmD \W_\B [  \overline{ \grad_\B \, h_{r+}([\U, \B])}  ]  ) \theta_B )\\
& \overset{ \eqref{eq: FOSP-condition-PSD2} }= \nabla^2 f(\U \B \U^\top)[\U \B \theta_U^\top + \U \theta_B \U^\top + \theta_U \B \U^\top, \U \B \theta_U^\top + \U \theta_B \U^\top + \theta_U \B \U^\top]  \\
& \quad  + 2 \langle  \nabla f(\U \B \U^\top), \theta_U \B \theta_U^\top \rangle\\
& \overset{ \text{Proposition } \ref{prop: psd-bijection2}, \eqref{eq: Hessian-con-gradient-2} }= \nabla^2 f(\X)[\cL_{\U,\B}^{r+}(\theta_{(\U,\B)}),\cL_{\U,\B}^{r+}(\theta_{(\U,\B)})] + 2\langle \nabla f(\X), P_{\U_\perp} \theta_U \B \B^{-1} \B \theta_U^\top  P_{\U_\perp}    \rangle\\
& = \Hess f(\X)[\cL_{\U,\B}^{r+}(\theta_{(\U,\B)}),\cL_{\U,\B}^{r+}(\theta_{(\U,\B)})],
\end{split}
\end{equation*}

where the last equality follows from the expression of  $\Hess f(\X)$ in (<ref>) and the definition of $\cL_{\U,\B}^{r+}$.

Then,  by (<ref>), (<ref>) and Theorem <ref>, we have $\overline{\Hess \, h_{r+}([\U,\B])}$ has $(pr-(r^2-r)/2)$ eigenvalues and $\widebar{\lambda}_i(\Hess \, h_{r+}([\U,\B]))$ is sandwiched between $(\sigma^2_r(\W_\B^{-1}) \wedge 2 \sigma_r^2(\V_\B^{-1/2} \B) ) \lambda_i(\Hess f(\X))$ and $(\sigma^2_1(\W^{-1}_\B) \vee 2 \sigma^2_1( \V_\B^{-1/2} \B ) ) \lambda_i(\Hess f(\X))$ for $i = 1,\ldots,pr-(r^2-r)/2$.   $\blacksquare$

§.§ Proof of Corollary <ref>.

Here we prove the Riemannian FOSP, SOSP, and strict saddle equivalence on $\cM_{r+}^e$ and $\cM_{r+}^{q_1}$, similar proof applies to the equivalence of (<ref>) on $\cM_{r+}^e$ and $\cM_{r+}^{q_2}$. First, by the connection of Riemannian gradients in (<ref>), the connection of Riemannian FOSPs under two geometries clearly holds.

Suppose $[\Y]$ is a Riemannian SOSP of (<ref>) and let $\X = \Y \Y^\top$. Given any  $\xi_\X \in T_{\X}\cM^e_{r+}$, we have $\Hess f(\X)[\xi_\X, \xi_\X] \overset{ \eqref{eq: Hessian-connection-PSD1} }=\overline{\Hess \, h_{r+}([\Y])}[(\cL^{r+}_\Y)^{-1}(\xi_\X), (\cL^{r+}_\Y)^{-1}(\xi_\X)] \geq 0$, where the inequality is by the SOSP assumption on $[\Y]$. Combining the fact $\X$ is a Riemannian FOSP, this shows $\X = \Y \Y^\top$ is a Riemannian SOSP under the embedded geometry.

Next, let us show the other direction: suppose $\X$ is a Riemannian SOSP under the embedded geometry, there is a unique $[\Y]$ such that $\Y \Y^\top = \X$ and it is a Riemannian SOSP of (<ref>). To see this, first the uniqueness of $[\Y]$ is guaranteed by the fact $\bar{\ell}: \Y \in \widebar{\cM}_{r+}^{q_1} \to  \Y \Y^\top \in \cM_{r+}^e $ induces a diffeomorphism between $\cM_{r+}^e$ and $\cM_{r+}^{q_1}$ <cit.>. In addition, we have shown $[\Y]$ is a Riemannian FOSP of (<ref>). Then by (<ref>), we have for any $\theta_\Y \in \cH_\Y \widebar{\cM}_{r+}^{q_1}$, $\overline{\Hess \, h_{r+}([\Y])}[\theta_\Y, \theta_\Y]  = \Hess f(\X)[\cL_\Y^{r+}(\theta_\Y),\cL_\Y^{r+}(\theta_\Y)] \geq 0$.

Finally, the equivalence on strict saddles also follows easily from the sandwich inequality from Theorem <ref> and the definition of the strict saddle.   $\blacksquare$

§ ADDITIONAL PROOFS IN SECTION <REF>

§.§ Proof of Proposition <ref>.

The proof is divided into five steps: in Step 1, we derive the expressions for Riemannian gradients; in Step 2, we derive the Riemannian Hessian of (<ref>) under the embedded geometry; in Step 3, we derive the Riemannian Hessian for $h_{r}([\L,\R])$; in Step 4, we derive the Riemannian Hessian for $h_{r}([\U,\B,\V])$; in Step 5, we derive the Riemannian Hessian for $h_r([\U,\Y])$.

Step 1.  First, the Riemannian gradient expression of (<ref>) under the embedded geometry can be found in <cit.>. Next, we compute $\grad \, \bar{h}_{r}(\L,\R)$, $\grad \, \bar{h}_{r}(\U, \B,\V)$ and $\grad \, \bar{h}_{r}(\U, \Y)$ from their definitions:

\begin{equation} \label{eq: quotient-gradient-comp-general-eq1}
\begin{split}
& \quad \langle \grad_{\L} \, \bar{h}_r(\L,\R) \W_{\L,\R}, \eta_L \rangle + \langle \grad_{\R} \, \bar{h}_r(\L,\R) \V_{\L,\R}, \eta_R \rangle \\
&= \bar{g}_{(\L,\R)}^r(\grad \, \bar{h}_{r}(\L,\R), \eta_{(\L,\R)} )\\
&= \rmD \bar{h}_{r}(\L,\R)[\eta_{(\L,\R)}] = \langle \nabla_{\L} \bar{h}_r(\L,\R) , \eta_L \rangle +\langle \nabla_{\R} \bar{h}_r(\L,\R) , \eta_R \rangle \\
&= \langle \nabla f(\L \R^\top) \R , \eta_L \rangle +\langle (\nabla f(\L \R^\top))^\top \L , \eta_R \rangle, \quad \forall \eta_{(\L,\R)} \in T_{(\L,\R)} \widebar{\cM}_r^{q_1},
\end{split}
\end{equation}
where $\nabla_{\L} \bar{h}_r(\L,\R)$ denotes the Euclidean gradient of $\bar{h}_r(\L,\R)$ with respect to $\L$,

\begin{equation}\label{eq: quotient-gradient-comp-general-eq2}
\begin{split}
&\langle \grad_\U \, \bar{h}_r(\U,\B,\V), \eta_U \rangle + \langle \B^{-1} \grad_{\B} \, \bar{h}_r(\U,\B,\V) \B^{-1}, \eta_B \rangle + \langle \grad_{\V} \, \bar{h}_r(\U,\B,\V), \eta_V \rangle\\
= & \bar{g}_{(\U,\B,\V)}^r (\grad \, \bar{h}_r(\U,\B,\V), \eta_{(\U,\B,\V)} ) = \rmD \bar{h}_r(\U,\B,\V)[\eta_{(\U,\B,\V)}] \\
= & \langle \nabla_{\U} \bar{h}_r(\U,\B,\V) , \eta_U \rangle +\langle \nabla_{\B} \bar{h}_r(\U,\B,\V) , \eta_B \rangle + \langle \nabla_{\V} \bar{h}_r(\U,\B,\V) , \eta_V \rangle \\
= & \langle \nabla f(\U \B \V^\top) \V \B , \eta_U \rangle +\langle \U^\top \nabla f(\U \B \V^\top) \V, \eta_B \rangle \\
&+ \langle (\nabla f(\U \B \V^\top))^\top \U \B , \eta_V \rangle, \quad \forall \eta_{(\U,\B,\V)} \in T_{(\U,\B,\V)} \widebar{\cM}_r^{q_2},
\end{split}
\end{equation}

\begin{equation}\label{eq: quotient-gradient-comp-general-eq3}
\begin{split}
& \quad \langle \grad_\U \, \bar{h}_r(\U,\Y) \V_\Y, \eta_U \rangle + \langle \grad_\Y \, \bar{h}_r(\U,\Y) \W_\Y, \eta_Y \rangle\\
&= \bar{g}_{(\U,\Y)}^r (\grad \, \bar{h}_r(\U,\Y), \eta_{(\U,\Y)} ) =\rmD \bar{h}_r(\U,\Y)[\eta_{(\U,\Y)}] \\
& = \langle \nabla_\U \bar{h}_r(\U,\Y) , \eta_U \rangle +\langle \nabla_\Y \bar{h}_r(\U,\Y) , \eta_Y \rangle\\
& = \langle \nabla f(\U \Y^\top) \Y , \eta_U \rangle +\langle (\nabla f(\U \Y^\top))^\top \U  , \eta_Y \rangle, \quad \forall \eta_{(\U,\Y)} \in T_{(\U,\Y)} \widebar{\cM}_r^{q_3}.
\end{split}
\end{equation}

From (<ref>) and (<ref>), we have

\begin{equation} \label{eq: RGradient-total-space-general}
\begin{split}
\grad \, \bar{h}_{r}(\L,\R) &=\begin{bmatrix}
\nabla f(\L \R^\top) \R \W_{\L,\R}^{-1} \\
(\nabla f(\L \R^\top))^\top \L \V_{\L,\R}^{-1}
\end{bmatrix},   \\
\grad \, \bar{h}_r(\U,\B,\V) &= \begin{bmatrix}
P_{ T_\U \st(r,p_1) } \left(  \nabla f(\U \B \V^\top) \V \B \right)\\
\B\sym( \U^\top \nabla f(\U \B \V^\top) \V )\B\\
P_{ T_\V \st(r,p_2) } \left( (\nabla f(\U \B \V^\top))^\top \U \B  \right)
\end{bmatrix}.
\end{split}
\end{equation}

Moreover, from (<ref>), we know $\grad_\Y \, \bar{h}_r(\U,\Y)= (\nabla f(\U \Y^\top))^\top \U \W_\Y^{-1}$. Suppose $\grad_\U \, \bar{h}_r(\U,\Y) = \U_\perp \D + \U \bOmega \in T_\U \st(r,p_1)$. Then from (<ref>), we have $P_{T_\U \st(r,p_1)}(  \grad_\U \, \bar{h}_r(\U,\Y) \V_\Y -  \nabla f(\U \Y^\top) \Y  ) = 0$. This yields $\D = \U_\perp^\top \nabla f(\U \Y^\top) \Y \V_\Y^{-1} $ and $\bOmega$ is uniquely determined by the Sylvester equation: $\V_\Y \bOmega^\top + \bOmega^\top \V_\Y = \Y^\top (\nabla f(\U \Y^\top))^\top \U - \U^\top \nabla f(\U \Y^\top) \Y  $. So

\begin{equation*}
\grad \, \bar{h}_r(\U,\Y) = \begin{bmatrix}
P_{\U_\perp}\nabla f(\U \Y^\top) \Y \V_\Y^{-1}  + \U \bOmega  \\
(\nabla f(\U \Y^\top))^\top \U \W_\Y^{-1}
\end{bmatrix}.
\end{equation*}

By Lemma <ref>, $\cV_{(\L,\R)} \widebar{\cM}_{r}^{q_1} $ is orthogonal to $ \cH_{(\L,\R)} \widebar{\cM}_{r}^{q_1}$ with respect to $\bar{g}^{r}_{(\L,\R)}$, we have $\overline{\grad\, h_{r}([\L,\R])} = \grad \, \bar{h}_{r}(\L,\R)$ by Lemma <ref>.

Next, we compute $\overline{\grad\, h_{r}([\U,\B,\V])}$. Given $\eta_{(\U,\B, \V)} = [(\U \bOmega_1 + \U_\perp \D_1)^\top \quad \S \quad  ( \V \bOmega_2 + \V_\perp \D_2)^\top]^\top\in T_{(\U,\B,\V)} \widebar{\cM}_r^{q_2}$, and suppose $P_{(\U,\B,\V)}^\cH (\eta_{(\U,\B, \V)}) = [ (\U \widetilde{\bOmega} + \U_\perp \widetilde{\D}_1)^\top \quad \widetilde{\S} \quad (-\V \widetilde{\bOmega} + \V_\perp \widetilde{\D}_2)^\top ] \in \cH_{(\U,\B,\V)} \widebar{\cM}_r^{q_2}$. By definition

\begin{equation*}
\begin{split}
&(\widetilde{\bOmega}, \widetilde{\S}, \widetilde{\D}_1, \widetilde{\D}_2) \\
= &\argmin_{ \substack{\bOmega' = -\bOmega^{'\top} \in \bbR^{r\times r}, \S' \in \bbS^{r \times r},\\ \D_1' \in \bbR^{(p_1 - r) \times r}, \D_2' \in \bbR^{(p_2 - r) \times r}} } \bar{g}_{(\U,\B,\V)}^r \left( \begin{bmatrix}
\U (\bOmega_1 - \bOmega') + \U_\perp (\D_1 - \D_1') \\
\S-\S' \\
\V (\bOmega_2 + \bOmega') + \V_\perp (\D_2 - \D_2')
\end{bmatrix}, \begin{bmatrix}
\U (\bOmega_1 - \bOmega') + \U_\perp (\D_1 - \D_1') \\
\S-\S' \\
\V (\bOmega_2 + \bOmega') + \V_\perp (\D_2 - \D_2')
\end{bmatrix}\right) \\
= &\argmin_{ \substack{\bOmega' = -\bOmega^{'\top} \in \bbR^{r\times r}, \S' \in \bbS^{r \times r},\\ \D_1' \in \bbR^{(p_1 - r) \times r}, \D_2' \in \bbR^{(p_2 - r) \times r}} } \|\bOmega_1 - \bOmega'\|^2_\F + \|\bOmega_2 + \bOmega'\|_\F^2 + \sum_{i=1}^2\| \D_i - \D_i'\|_\F^2  + \|\B^{-1/2} (\S - \S') \B^{-1/2} \|_\F^2.
\end{split}
\end{equation*}
So we have $(\widetilde{\bOmega}, \widetilde{\S}, \widetilde{\D}_1, \widetilde{\D}_2) = ( (\bOmega_1 - \bOmega_2)/2, \S, \D_1, \D_2)$, i.e.,

\begin{equation} \label{eq: horizontal-projection-UBV}
P_{(\U,\B,\V)}^\cH(\eta_{(\U,\B,\V)}) = \begin{bmatrix}
P_{\U_\perp} \eta_U + \U(\U^\top \eta_U - \V^\top \eta_V )/2\\
\eta_B \\
P_{\V_\perp} \eta_V - \V(\U^\top \eta_U - \V^\top \eta_V )/2
\end{bmatrix}, \quad \eta_{(\U,\B,\V)} \in T_{(\U,\B,\V)} \widebar{\cM}_r^{q_2}.
\end{equation}

Then recall the definition of the projection operator $P_{ T_\U \st(r,p_1) }$ from Table <ref>, we have

\begin{equation*}
\begin{split}
\overline{\grad\, h_{r}([\U,\B,\V])} \overset{ \eqref{eq: quotient-gradient-general} } =& P_{(\U,\B,\V)}^\cH \left( \grad \, \bar{h}_r(\U,\B,\V) \right)
= \begin{bmatrix}
P_{\U_\perp} \nabla f(\U \B \V^\top) \V \B + \U \bOmega \\
\B \sym(\U^\top \nabla f(\U \B \V^\top) \V) \B \\
P_{\V_\perp} (\nabla f(\U \B \V^\top))^\top \U \B - \V \bOmega
\end{bmatrix},
\end{split}
\end{equation*}

\begin{equation*}
\begin{split}
\bOmega &= \left(\skew\left( \U^\top \nabla f(\U \B \V^\top) \V \B  \right) - \skew\left(\V^\top (\nabla f(\U \B \V^\top))^\top \U \B \right)\right)/2\\
& = \skew(\U^\top \nabla f(\U \B \V^\top) \V ) \B/2 + \B \skew(\U^\top \nabla f(\U \B \V^\top) \V )/2.
\end{split}
\end{equation*}

\begin{equation*}
\begin{split}
\overline{\grad\, h_{r}([\U,\Y])} \overset{ \eqref{eq: quotient-gradient-general} } = P_{(\U,\V)}^{\cH}( \grad \, \bar{h}_r(\U,\Y) )  &=\begin{bmatrix}
P_{\U_\perp}\left( P_{\U_\perp}\nabla f(\U \Y^\top) \Y \V_\Y^{-1}  + \U \bOmega  \right)\\
(\nabla f(\U \Y^\top))^\top \U \W_\Y^{-1}
\end{bmatrix}\\
& = \begin{bmatrix}
P_{\U_\perp} \nabla f(\U \Y^\top) \Y \V^{-1}_\Y \\
(\nabla f(\U \Y^\top))^\top \U \W_\Y^{-1}
\end{bmatrix}.
\end{split}
\end{equation*}

Step 2. Suppose $\X$ has has SVD $\U' \bSigma' \V^{'\top}$ with $\U = \U' \O_1,\V = \V' \O_2$ ($\O_1,\O_2 \in \bbO_r$). Then for

\begin{equation*}
\begin{split}
\xi_\X &= [\U \quad \U_\perp] \begin{bmatrix}
\S & \D_2^\top\\
\D_1 & \0
\end{bmatrix} [\V \quad \V_\perp]^\top = [\U' \quad \U_\perp] \begin{bmatrix}
\O_1\S\O_2^\top & \O_1\D_2^\top\\
\D_1 \O_2^\top & \0
\end{bmatrix} [\V' \quad \V_\perp]^\top,
\end{split}
\end{equation*}
by <cit.> we have in the general case

\begin{equation*}
\begin{split}
\Hess f(\X)[\xi_\X, \xi_\X] &= \nabla^2 f(\X)[\xi_\X, \xi_\X] + 2\langle \nabla f(\X), \U_\perp \D_1 \O_2^\top \bSigma^{'-1} \O_1 \D_2^\top \V_\perp^\top \rangle \\
& =  \nabla^2 f(\X)[\xi_\X, \xi_\X] + 2\langle \nabla f(\X), \U_\perp \D_1 \bSigma^{-1} \D_2^\top \V_\perp^\top \rangle,
\end{split}
\end{equation*}
where $\bSigma = \O_1^\top \bSigma' \O_2 = \U^\top \X \V$.

Step 3. For convenience, we denote $\bar{g}_{\L}^{r}(\eta_L, \theta_L) = \tr(\W_{\L,\R}\eta_L^\top \theta_L) $ as the $\L$ part inner product of $\bar{g}^r_{(\L, \R)}$ and $\bar{g}_{\R}^{r}(\eta_R, \theta_R) = \tr(\V_{\L,\R}\eta_R^\top \theta_R)$ as the $\R$ part inner product of $\bar{g}^r_{(\L, \R)}$. First following similar arguments as in (<ref>) and (<ref>), for three vector fields $\eta, \theta, \xi$ on $\widebar{\cM}_r^{q_1}$, we have

\begin{equation*} \label{eq: directional-derivative-gLR}
\begin{split}
\rmD \bar{g}_{(\L,\R)} (\eta, \theta)[\xi] = \tr(\rmD \W_{\L,\R} [\xi] \eta_L^\top \theta_L ) + \tr(\rmD \V_{\L,\R} [\xi] \eta_R^\top \theta_R ) +\bar{g}_{(\L,\R)}( \rmD \eta[\xi], \theta ) + \bar{g}_{(\L,\R)}( \eta, \rmD \theta[\xi] )
\end{split}
\end{equation*}

\begin{equation} \label{eq: koszul-formula-gLR}
\begin{split}
2 \bar{g}_{(\L,\R)} (\widebar{\nabla}_\xi \eta, \theta) &= 2 \bar{g}_{(\L,\R)} ( \rmD \eta[\xi], \theta ) + \tr( \rmD \W_{\L,\R} [\xi] \eta_L^\top \theta_L ) + \tr( \rmD \V_{\L,\R} [\xi] \eta_R^\top \theta_R ) + \tr( \rmD \W_{\L,\R} [\eta] \xi_L^\top \theta_L )\\
& \quad + \tr( \rmD \V_{\L,\R} [\eta] \xi_R^\top \theta_R )- \tr( \rmD \W_{\L,\R} [\theta] \eta_L^\top \xi_L ) - \tr( \rmD \V_{\L,\R} [\theta] \eta_R^\top \xi_R ).
\end{split}
\end{equation}

\begin{equation} \label{eq: koszul-formula-LR}
\begin{split}
&\quad \overline{\Hess \, h_{r}([\L,\R])}[\theta_{(\L,\R)}, \theta_{(\L,\R)}] \\
&= \bar{g}_{(\L,\R)}^{r}\left( \overline{\Hess \, h_{r}([\L,\R])[\theta_{[\L,\R]}]} , \theta_{(\L,\R)} \right)\\
&\overset{ \eqref{eq: quotient-hessian-linear-from} } =\bar{g}_{(\L,\R)}^{r} \left( P_{(\L,\R)}^{\cH} \left( \widebar{\nabla}_{\theta_{(\L,\R)}} \overline{ \grad \, h_{r}([\L,\R]) } \right) , \theta_{(\L,\R)} \right)\\
&\overset{ \text{Lemma } \ref{lm: quotient-hessian-riemannian-connection-equal} } = \bar{g}_{(\L,\R)}^{r} \left( \widebar{\nabla}_{\theta_{(\L,\R)}} \overline{ \grad \, h_{r}([\L,\R])} , \theta_{(\L,\R)} \right) \\
& \overset{ \eqref{eq: koszul-formula-gLR} } = \bar{g}_{\L}^{r} \left( \rmD \overline{ \grad_{\L} \, h_{r}([\L,\R])}[\theta_{(\L,\R)}] , \theta_{L} \right) + \bar{g}_{\R}^{r} \left( \rmD \overline{ \grad_{\R} \, h_{r}([\L,\R])}[\theta_{(\L, \R)}], \theta_R \right)\\
& \quad  + \tr( \rmD \W_{\L,\R}[ \overline{ \grad \, h_{r}([\L,\R])} ] \theta_L^\top \theta_L )/2 +  \tr( \rmD \V_{\L,\R}[ \overline{ \grad \, h_{r}([\L,\R])} ] \theta_R^\top \theta_R )/2.
\end{split}
\end{equation}

Next, we compute $\rmD \overline{ \grad_{\L} \, h_{r}([\L,\R])}[\theta_{(\L,\R)}]$.

\begin{equation} \label{eq: directional-derivative-L}
\begin{split}
&\rmD \overline{ \grad_{\L} \, h_{r}([\L,\R])}[\theta_{(\L,\R)}] \\
=& \lim_{t \to 0} \left( \nabla f\left((\L+ t\theta_L) (\R + t\theta_R )^\top\right) (\R + t\theta_R ) \W_{\L+ t\theta_L,\R + t\theta_R}^{-1} -\nabla f(\L \R^\top) \R \W_{\L,\R}^{-1}  \right)/t \\
=& \nabla f(\L \R^\top) \theta_R \W_{\L,\R}^{-1} + \nabla f(\L \R^\top) \R \rmD \W_{\L,\R}^{-1}[\theta_{(\L,\R)}] + \nabla^2 f(\L \R^\top)[\L \theta_R^\top + \theta_L \R^\top] \R  \W_{\L,\R}^{-1}.
\end{split}
\end{equation}
Similarly we have

\begin{equation} \label{eq: directional-derivative-R}
\begin{split}
&\rmD \overline{ \grad_{\R} \, h_{r}([\L,\R])}[\theta_{(\L, \R)}] \\
=& \left(\nabla f(\L \R^\top) \right)^\top \theta_L \V_{\L,\R}^{-1} + \left(\nabla f(\L \R^\top) \right)^\top \L \rmD \V_{\L,\R}^{-1}[\theta_{(\L,\R)}] + \left(\nabla^2 f(\L \R^\top)[\L \theta_R^\top + \theta_L \R^\top]\right)^\top \L  \V_{\L,\R}^{-1}.
\end{split}
\end{equation}

Plugging (<ref>) and (<ref>) into (<ref>), we have

\begin{equation*}
\begin{split}
&\quad \overline{\Hess \, h_{r}([\L,\R])}[\theta_{(\L,\R)}, \theta_{(\L,\R)}] \\
&=  \nabla^2 f(\L \R^\top)[\L \theta_R^\top + \theta_L \R^\top,\L \theta_R^\top + \theta_L \R^\top] + 2 \langle \nabla f(\L \R^\top), \theta_L \theta_R^\top \rangle \\
& \quad + \langle \nabla f(\L \R^\top) \R \rmD \W_{\L,\R}^{-1}[\theta_{(\L,\R)}] , \theta_L \W_{\L,\R} \rangle + \langle  \left(\nabla f(\L \R^\top) \right)^\top \L \rmD \V_{\L,\R}^{-1}[\theta_{(\L,\R)}], \theta_R \V_{\L,\R} \rangle \\
& \quad +  \tr( \rmD \W_{\L,\R}[ \overline{ \grad \, h_{r}([\L,\R])} ] \theta_L^\top \theta_L )/2 +  \tr( \rmD \V_{\L,\R}[ \overline{ \grad \, h_{r}([\L,\R])} ] \theta_R^\top \theta_R )/2.
\end{split}
\end{equation*}

Step 4. For convenience, we denote $\bar{g}_{\U}^{r}(\eta_U, \theta_U) = \tr(\eta_U^\top \theta_U) $ as the $\U$ part inner product of $\bar{g}^r_{(\U, \B,\V)}$, $\bar{g}^r_{\B}(\eta_B, \theta_B) = \tr(\B^{-1} \eta_B \B^{-1} \theta_B)$ as the $\B$ part inner product of $\bar{g}^r_{(\U, \B,\V)}$ and $\bar{g}_{\V}^{r}(\eta_V, \theta_V) = \tr(\eta_V^\top \theta_V) $ as the $\V$ part inner product of $\bar{g}^r_{(\U, \B,\V)}$. Then

\begin{equation} \label{eq: quotient-hessian-UBV}
\begin{split}
& \quad \overline{\Hess \, h_{r}([\U,\B,\V])}[\theta_{(\U,\B,\V)}, \theta_{(\U,\B,\V)}] \\
&= \bar{g}_{(\U,\B,\V)}^{r}\left( \overline{\Hess \, h_{r}([\U,\B,\V])[\theta_{[\U,\B,\V]}]} , \theta_{(\U,\B,\V)} \right)\\
& \overset{\eqref{eq: quotient-hessian-linear-from}, \text{Lemma } \ref{lm: quotient-hessian-riemannian-connection-equal} } = \bar{g}_{(\U,\B,\V)}^{r} \left( \widebar{\nabla}_{\theta_{(\U,\B,\V)}} \overline{ \grad \, h_{r}([\U,\B,\V])} , \theta_{(\U, \B,\V)} \right)\\
& \overset{ \eqref{eq: koszul-formula-simple-manifolds} } = \bar{g}_{\U}^{r} \left( \rmD \overline{ \grad_\U \, h_{r}([\U, \B,\V])}[\theta_{(\U, \B,\V)}] , \theta_{U} \right) + \bar{g}_{\B}^{r} \left( \rmD \overline{ \grad_\B \, h_{r}([\U, \B,\V])}[\theta_{(\U, \B,\V)}], \theta_B \right)\\
& \quad - \tr\left(\B^{-1} \sym\left( \theta_B \B^{-1} \overline{ \grad_\B \, h_{r}([\U, \B,\V])} \right) \B^{-1} \theta_B \right) + \bar{g}_{\V}^{r} \left( \rmD \overline{ \grad_\V \, h_{r}([\U, \B,\V])}[\theta_{(\U, \B,\V)}] , \theta_{V} \right).
\end{split}
\end{equation}

Recall the horizontal projection operator $P_{(\U,\B,\V)}^\cH$ given in (<ref>), and let $P_{\U}^\cH$, $P_{\B}^\cH$ and $P_{\V}^\cH$ be the restriction of $P_{(\U,\B,\V)}^\cH$ to the $\U$, $\B$ and $\V$ components, respectively. Moreover, define $\U_t = \U + t \theta_U, \V_t = \V + t \theta_V, \B_t = \B + t\theta_B$. By (<ref>), we have

\begin{equation} \label{eq: directional-derivative-gradient-UBV}
\begin{split}
\bDelta_U:=&\lim_{t \to 0} \left( \grad_\U \, \bar{h}_r(\U_t,\B_t,\V_t) -  \grad_\U \, \bar{h}_r(\U,\B,\V))  \right)/t\\
=& 	P_{ T_\U \st(r,p_1) } \left(  \nabla f(\U \B \V^\top) \theta_V \B + \nabla f(\U \B \V^\top) \V \theta_B + \nabla^2 f(\U \B \V^\top)[\theta_U \B \V^\top + \U \theta_B \V^\top + \U \B \theta_V^\top] \V \B  \right)\\
& - 2 \sym(\U \theta_U^\top) \nabla f(\U \B \V^\top) \V \B + \theta_U \skew\left(\U^\top\nabla f(\U \B \V^\top) \V \B \right) + \U \skew\left(  \theta_U^\top\nabla f(\U \B \V^\top) \V \B \right);\\
\bDelta_V:= &\lim_{t \to 0} \left( \grad_\V \, \bar{h}_r(\U_t,\B_t,\V_t) -  \grad_\V \, \bar{h}_r(\U,\B,\V))  \right)/t\\
=& 	P_{ T_\V \st(r,p_2) } \Big(  \left(\nabla f(\U \B \V^\top)\right)^\top \theta_U \B + \left(\nabla f(\U \B \V^\top)\right)^\top \U \theta_B \\
&+ \left(\nabla^2 f(\U \B \V^\top)[\theta_U \B \V^\top + \U \theta_B \V^\top + \U \B \theta_V^\top]\right)^\top \U \B  \Big)- 2 \sym(\V \theta_V^\top) \left(\nabla f(\U \B \V^\top)\right)^\top \U \B\\
& + \theta_V \skew\left(\V^\top\left(\nabla f(\U \B \V^\top)\right)^\top \U \B \right) + \V \skew\left(  \theta_V^\top\left(\nabla f(\U \B \V^\top)\right)^\top \U \B \right).
\end{split}
\end{equation}

\begin{equation} \label{eq: U-part-UBV-directional-derivative}
\begin{split}
&\rmD \overline{ \grad_\U \, h_{r}([\U, \B,\V])}[\theta_{(\U, \B,\V)}] \\
=& \lim_{t \to 0} \Big(  P_{\U_t}^\cH (  \grad_\U \, \bar{h}_r(\U_t,\B_t,\V_t) ) - P_\U^\cH ( \grad_\U \, \bar{h}_r(\U,\B,\V)) \Big)/t \\
=& P_\U^\cH \left( \lim_{t \to 0} \left( \grad_\U \, \bar{h}_r(\U_t,\B_t,\V_t) -  \grad_\U \, \bar{h}_r(\U,\B,\V)  \right)/t  \right) - \sym(\U \theta_U^\top) \grad_\U \, \bar{h}_r(\U,\B,\V) \\
& - (\theta_U \V^\top + \U \theta_V^\top) \grad_\V \, \bar{h}_r(\U,\B,\V)/2 \\
\overset{ \eqref{eq: directional-derivative-gradient-UBV} } = & P_\U^\cH (\bDelta_U)  - \sym(\U \theta_U^\top) \grad_\U \, \bar{h}_r(\U,\B,\V) - (\theta_U \V^\top + \U \theta_V^\top) \grad_\V \, \bar{h}_r(\U,\B,\V)/2,
\end{split}
\end{equation}

\begin{equation} \label{eq: V-part-UBV-directional-derivative}
\begin{split}
&\rmD \overline{ \grad_\V \, h_{r}([\U, \B,\V])}[\theta_{(\U, \B,\V)}] \\
=& \lim_{t \to 0} \Big(  P_{\V_t}^\cH (  \grad_\V \, \bar{h}_r(\U_t,\B_t,\V_t) ) - P_\V^\cH ( \grad_\V \, \bar{h}_r(\U,\B,\V)) \Big)/t \\
=& P_\V^\cH \left( \lim_{t \to 0} \left( \grad_\V \, \bar{h}_r(\U_t,\B_t,\V_t) -  \grad_\V \, \bar{h}_r(\U,\B,\V)  \right)/t  \right) - \sym(\V \theta_V^\top) \grad_\V \, \bar{h}_r(\U,\B,\V) \\
& - (\theta_V \U^\top + \V \theta_U^\top) \grad_\V \, \bar{h}_r(\U,\B,\V)/2 \\
\overset{ \eqref{eq: directional-derivative-gradient-UBV} } = & P_\U^\cH (\bDelta_V)  - \sym(\V \theta_V^\top) \grad_\V \, \bar{h}_r(\U,\B,\V) - (\theta_V \U^\top + \V \theta_U^\top) \grad_\U   \, \bar{h}_r(\U,\B,\V)/2.
\end{split}
\end{equation}

In addition

\begin{equation} \label{eq: B-part-UBV-directional-derivative}
\begin{split}
&\rmD \overline{ \grad_\B \, h_{r}([\U, \B,\V])}[\theta_{(\U, \B,\V)}] \\
=& \lim_{t \to 0} \Big(   \grad_\B \, \bar{h}_r(\U_t,\B_t,\V_t)  - \grad_\B \, \bar{h}_r(\U,\B,\V) \Big)/t \\
=& \B\Big( \sym\left(\theta_U^\top \nabla f(\U \B \V^\top) \V \right) + \sym\left(\U^\top \nabla f(\U \B \V^\top) \theta_V\right) \\
&+ \U^\top  \nabla^2 f(\U \B \V^\top)[\theta_U \B \V^\top + \U \theta_B \V^\top + \U \B \theta_V^\top] \V   \Big) \B \\
& + \B\sym(\U^\top \nabla f(\U \B \V^\top) \V ) \theta_B + \theta_B \sym(\U^\top \nabla f(\U \B \V^\top) \V ) \B.
\end{split}
\end{equation}

Let $\bDelta =  \U^\top\nabla f(\U \B \V^\top) \V$, $\bDelta' =  \theta_U^\top\nabla f(\U \B \V^\top) \V$ and $\bDelta'' =  \U^\top\nabla f(\U \B \V^\top) \theta_V$. By (<ref>), (<ref>) and (<ref>) and observing the fact $\theta_{(\U,\B, \V)} \in \cH_{(\U,\B,\V)} \widebar{\cM}_r^{q_2}$, we have

\begin{equation} \label{eq: U-part-UBV-directional-derivative2}
\begin{split}
&\bar{g}_{\U}^{r} \left( \rmD \overline{ \grad_\U \, h_{r}([\U, \B,\V])}[\theta_{(\U, \B,\V)}] , \theta_{U} \right) \\
=& \left\langle \bDelta_U  - \sym(\U \theta_U^\top) \grad_\U \, \bar{h}_r(\U,\B,\V) - (\theta_U \V^\top + \U \theta_V^\top) \grad_\V \, \bar{h}_r(\U,\B,\V)/2, \theta_U  \right\rangle \\
=&  \left\langle \bDelta_U , \theta_U  \right\rangle+ \langle \bDelta, \sym(\U^\top \theta_U \U^\top\theta_U) \B + \B \sym(\V^\top \theta_V \U^\top \theta_U) \rangle/2 - \langle \bDelta', \U^\top \theta_U \B \rangle/2 - \langle \bDelta'', \B \theta_U^\top \U \rangle/2;\\
\overset{\eqref{eq: directional-derivative-gradient-UBV} } =&  \nabla^2 f(\U \B \V^\top)[\theta_U \B \V^\top + \U \theta_B \V^\top + \U \B \theta_V^\top, \theta_U \B \V^\top] + \langle \nabla f(\U\B\V^\top), \theta_U \B \theta_V^\top \rangle  \\
& - \langle \bDelta, \theta_U^\top \theta_U \B \rangle + \langle \bDelta', 2\theta_B -  \U^\top \theta_U \B - \theta_U^\top \U \B \rangle/2 +   \langle \bDelta, \sym(\U^\top \theta_U \U^\top\theta_U) \B + \B \sym(\V^\top \theta_V \U^\top \theta_U) \rangle/2  \\
& - \langle \bDelta', \U^\top \theta_U \B \rangle/2 - \langle \bDelta'', \B \theta_U^\top \U \rangle/2\\
= & \nabla^2 f(\U \B \V^\top)[\theta_U \B \V^\top + \U \theta_B \V^\top + \U \B \theta_V^\top, \theta_U \B \V^\top] + \langle \nabla f(\U\B\V^\top), \theta_U \B \theta_V^\top \rangle  \\
& + \left\langle \bDelta,  \sym(\U^\top \theta_U \U^\top\theta_U) \B + \B \sym(\V^\top \theta_V \U^\top \theta_U) -2\theta_U^\top \theta_U \B \right\rangle/2 \\
& + \langle \bDelta', \theta_B -  \U^\top \theta_U \B - \theta_U^\top \U \B/2 \rangle - \langle \bDelta'', \B \theta_U^\top \U \rangle/2,
\end{split}
\end{equation}
and similarly

\begin{equation} \label{eq: V-part-UBV-directional-derivative2}
\begin{split}
& \bar{g}_{\V}^{r} \left( \rmD \overline{ \grad_\V \, h_{r}([\U, \B,\V])}[\theta_{(\U, \B,\V)}] , \theta_{V} \right)  \\
= & \nabla^2 f(\U \B \V^\top)[\theta_U \B \V^\top + \U \theta_B \V^\top + \U \B \theta_V^\top, \U \B \theta_V^\top] + \langle \nabla f(\U\B\V^\top), \theta_U \B \theta_V^\top \rangle\\
& + \left\langle \bDelta,  \B\sym(\V^\top \theta_V \V^\top\theta_V) + \sym(\U^\top \theta_U \V^\top \theta_V) \B -2\B\theta_V^\top \theta_V \right\rangle/2 \\
& + \langle \bDelta'', \theta_B -  \B\theta_V^\top \V - \B\V^\top \theta_V/2 \rangle - \langle \bDelta', \V^\top \theta_V \B \rangle/2,
\end{split}
\end{equation}

\begin{equation} \label{eq: B-part-UBV-directional-derivative2}
\begin{split}
& \bar{g}_{\B}^{r} \left( \rmD \overline{ \grad_\B \, h_{r}([\U, \B,\V])}[\theta_{(\U, \B,\V)}], \theta_B \right)  - \tr\left(\B^{-1} \sym\left( \theta_B \B^{-1} \overline{ \grad_\B \, h_{r}([\U, \B,\V])} \right) \B^{-1} \theta_B \right) \\
= & \nabla^2 f(\U \B \V^\top)[\theta_U \B \V^\top + \U \theta_B \V^\top + \U \B \theta_V^\top, \U \theta_B \V^\top] + 2 \langle \bDelta, \theta_B \B^{-1} \theta_B \rangle \\
& + \langle \bDelta', \theta_B \rangle + \langle \bDelta'', \theta_B \rangle - \langle \bDelta, \theta_B \B^{-1} \theta_B \rangle\\
= & \nabla^2 f(\U \B \V^\top)[\theta_U \B \V^\top + \U \theta_B \V^\top + \U \B \theta_V^\top, \U \theta_B \V^\top] + \langle \bDelta, \theta_B \B^{-1} \theta_B \rangle + \langle \bDelta', \theta_B \rangle + \langle \bDelta'', \theta_B \rangle.
\end{split}
\end{equation}
By plugging (<ref>), (<ref>) and (<ref>) into (<ref>), we finally have

\begin{equation*}
\begin{split}
&\overline{\Hess \, h_{r}([\U,\B,\V])}[\theta_{(\U,\B,\V)}, \theta_{(\U,\B,\V)}] \\
=& \bar{g}_{\U}^{r} \left( \rmD \overline{ \grad_\U \, h_{r}([\U, \B,\V])}[\theta_{(\U, \B,\V)}] , \theta_{U} \right) + \bar{g}_{\B}^{r} \left( \rmD \overline{ \grad_\B \, h_{r}([\U, \B,\V])}[\theta_{(\U, \B,\V)}], \theta_B \right)\\
& - \tr\left(\B^{-1} \sym\left( \theta_B \B^{-1} \overline{ \grad_\B \, h_{r}([\U, \B,\V])} \right) \B^{-1} \theta_B \right) + \bar{g}_{\V}^{r} \left( \rmD \overline{ \grad_\V \, h_{r}([\U, \B,\V])}[\theta_{(\U, \B,\V)}] , \theta_{V} \right)\\
=& \nabla^2 f(\U \B \V^\top)[\theta_U \B \V^\top + \U \theta_B \V^\top + \U \B \theta_V^\top, \theta_U \B \V^\top + \U \theta_B \V^\top + \U \B \theta_V^\top] + 2\langle \nabla f(\U\B\V^\top), \theta_U \B \theta_V^\top \rangle\\
& +   \left\langle \bDelta,  \sym(\U^\top \theta_U \U^\top\theta_U) \B + \B \sym(\V^\top \theta_V \U^\top \theta_U) -2\theta_U^\top \theta_U \B \right\rangle/2\\
& + \left\langle \bDelta,  \B\sym(\V^\top \theta_V \V^\top\theta_V) + \sym(\U^\top \theta_U \V^\top \theta_V) \B -2\B\theta_V^\top \theta_V +  2\theta_B \B^{-1} \theta_B \right\rangle/2\\
& + \langle \bDelta', 2\theta_B -  \U^\top \theta_U \B - \theta_U^\top \U \B/2 -\V^\top \theta_V \B/2 \rangle + \langle \bDelta'', 2\theta_B -  \B\theta_V^\top \V - \B\V^\top \theta_V/2 -\B \theta_U^\top \U /2 \rangle.
\end{split}
\end{equation*}

Step 5.  For convenience, we denote $\bar{g}_{\U}^{r}(\eta_U, \theta_U) = \tr(\V_\Y \eta_U^\top \theta_U) $ as the $\U$ part inner product of $\bar{g}^r_{(\U, \Y)}$ and $\bar{g}_{\Y}^{r}(\eta_Y, \theta_Y) = \tr(\W_\Y\eta_Y^\top \theta_Y)$ as the $\Y$ part inner product of $\bar{g}^r_{(\U, \Y)}$.  Then

\begin{equation} \label{eq: koszul-formula-UY}
\begin{split}
&\quad \overline{\Hess \, h_{r}([\U,\Y])}[\theta_{(\U,\Y)}, \theta_{(\U,\Y)}] \\
&= \bar{g}_{(\U,\Y)}^{r}\left( \overline{\Hess \, h_{r}([\U,\Y])[\theta_{[\U,\Y]}]} , \theta_{(\U,\Y)} \right)\\
&\overset{\eqref{eq: quotient-hessian-linear-from}, \text{Lemma } \ref{lm: quotient-hessian-riemannian-connection-equal} } = \bar{g}_{(\U,\Y)}^{r} \left( \widebar{\nabla}_{\theta_{(\U,\Y)}} \overline{ \grad \, h_{r}([\U,\Y])} , \theta_{(\U,\Y)} \right) \\
& \overset{ \eqref{eq: koszul-formula-simple-manifolds} } = \bar{g}_{\U}^{r} \left( \rmD \overline{ \grad_\U \, h_{r}([\U,\Y])}[\theta_{(\U,\Y)}] , \theta_{U} \right) + \bar{g}_{\Y}^{r} \left( \rmD \overline{ \grad_\Y \, h_{r}([\U,\Y])}[\theta_{(\U, \Y)}], \theta_Y \right)\\
& \quad +  \tr\left( \rmD \V_\Y\left[\overline{ \grad_\Y \, h_{r}([\U,\Y])}\right] \theta_U^\top \theta_U  \right)/2  +  \tr\left( \rmD \W_\Y\left[\overline{ \grad_\Y \, h_{r}([\U,\Y])}\right] \theta_Y^\top \theta_Y  \right)/2.
\end{split}
\end{equation}

Next, we compute $ \rmD \overline{ \grad_\U \, h_{r}([\U,\Y])}[\theta_{(\U,\Y)}]$ and $ \rmD \overline{ \grad_\Y \, h_{r}([\U,\Y])}[\theta_{(\U, \Y)}]$ separately. We have

\begin{equation} \label{eq: directional-derivative-UY-Upart}
\begin{split}
&\rmD \overline{ \grad_\U \, h_{r}([\U,\Y])}[\theta_{(\U,\Y)}] \\
=& \lim_{t \to 0} \left( (\I_{p_1} - P_{\U + t\theta_U}) \nabla f\left( (\U + t\theta_U) (\Y + t \theta_Y )^\top \right) (\Y+ t\theta_Y) \V^{-1}_{\Y + t \theta_Y } - P_{\U_\perp} \nabla f(\U \Y^\top) \Y \V^{-1}_\Y \right)/t\\
=&  P_{\U_\perp}\left( \nabla f(\U \Y^\top) \theta_Y \V^{-1}_\Y +  \nabla f(\U \Y^\top) \Y \rmD \V_\Y^{-1}[\theta_Y]  + \nabla^2 f(\U \Y^\top)[\U \theta_Y^\top + \theta_U \Y^\top] \Y \V^{-1}_\Y  \right)\\
& - (\U \theta_U^\top  + \theta_U \U^\top) \nabla f(\U \Y^\top) \Y\V^{-1}_\Y,
\end{split}
\end{equation}

\begin{equation}\label{eq: directional-derivative-UY-Ypart}
\begin{split}
&\rmD \overline{ \grad_\Y \, h_{r}([\U,\Y])}[\theta_{(\U, \Y)}] \\
= & \lim_{t \to 0} \left( (\nabla f\left((\U + t\theta_U) (\Y + t \theta_Y )^\top\right))^\top (\U + t\theta_U) \W_{(\Y + t \theta_Y )}^{-1}  - (\nabla f(\U \Y^\top))^\top \U \W_\Y^{-1} \right)/t\\
= & (\nabla f(\U \Y^\top))^\top \theta_U \W_\Y^{-1} + (\nabla f(\U \Y^\top))^\top \U \rmD \W_\Y^{-1} [\theta_Y] + \left( \nabla^2 f(\U \Y^\top)[\U \theta_Y^\top + \theta_U \Y^\top] \right)^\top \U \W_\Y^{-1}.
\end{split}
\end{equation}

By plugging (<ref>) and (<ref>) into (<ref>) and observing the fact $\theta_{(\U,\Y)} \in \cH_{(\U,\Y)} \widebar{\cM}_r^{q_3}$, we have

\begin{equation*}
\begin{split}
&\overline{\Hess \, h_{r}([\U,\Y])}[\theta_{(\U,\Y)}, \theta_{(\U,\Y)}] \\
=& \nabla^2 f(\U \Y^\top)[\U \theta_Y^\top + \theta_U \Y^\top, \U \theta_Y^\top + \theta_U \Y^\top] + 2 \langle  \nabla f(\U \Y^\top), \theta_U \theta_Y^\top \rangle - \langle  \U^\top \nabla f(\U \Y^\top) \Y, \theta_U^\top \theta_U \rangle \\
& + \langle (\nabla f(\U \Y^\top))^\top \U \rmD \W_\Y^{-1} [\theta_Y], \theta_Y \W_\Y \rangle + \langle \nabla f(\U \Y^\top) \Y \rmD \V_\Y^{-1} [\theta_Y], \theta_U \V_\Y \rangle\\
& + \tr\left( \rmD \V_\Y\left[\overline{ \grad_\Y \, h_{r}([\U,\Y])}\right] \theta_U^\top \theta_U  \right)/2 + \langle \rmD \W_\Y\left[\overline{ \grad_\Y \, h_{r}([\U,\Y])}\right], \theta_Y^\top \theta_Y  \rangle /2.
\end{split}
\end{equation*}
This finishes the proof of this proposition.   $\blacksquare$

§.§ Proof of Theorem <ref>.

First, recall $\X = \U \Y^\top$ and $\V$ spans the right singular subspace of $\X$, so $\Y$ lies in the column space of $\V$ and $\Y \Y^\dagger = P_\V$. Thus, (<ref>) is by direct calculation from the gradient expressions in Proposition <ref>.

Next, we prove (<ref>). Since $[\U,\Y]$ is a Riemannian FOSP of (<ref>), we have

\begin{equation} \label{eq: FOSP-condition-general3}
\overline{ \grad \, h_{r}([\U,\Y])} = \0 \quad, \quad    (\nabla f(\U \Y^\top))^\top \U = \0 \quad,  \text{ and } \quad  \nabla f(\U \Y^\top) \Y = \0
\end{equation}
and have $\grad f(\X) = \0$ by (<ref>). So $ \nabla f(\X) = P_{\U_\perp} \nabla f(\X) P_{\V_\perp}$. Let $\bSigma = \U^\top \X \V$. Given any $\theta_{(\U,\Y)} =  [\theta_U^\top \quad \theta_Y^\top]^\top \in \cH_{(\U,\Y)} \widebar{\cM}_{r}^{q_3}$, we have

\begin{equation} \label{eq: Hessian-con-gradient-general3}
\langle \nabla f(\X), P_{\U_\perp} \theta_U \Y^\top \V \bSigma^{-1}  \theta_Y^\top  P_{\V_\perp}    \rangle = \langle \nabla f(\X), \theta_U \theta_Y^\top \rangle,
\end{equation}
where the equality is because $\Y^\top \V = \U^\top \X \V = \bSigma$ and $ \nabla f(\X) = P_{\U_\perp} \nabla f(\X) P_{\V_\perp} $.

Then by Proposition <ref>:

\begin{equation*}
\begin{split}
&\quad \overline{\Hess \, h_{r}([\U,\Y])}[\theta_{(\U,\Y)}, \theta_{(\U,\Y)}] \\
=& \nabla^2 f(\U \Y^\top)[\U \theta_Y^\top + \theta_U \Y^\top, \U \theta_Y^\top + \theta_U \Y^\top] + 2 \langle  \nabla f(\U \Y^\top), \theta_U \theta_Y^\top \rangle - \langle  \U^\top \nabla f(\U \Y^\top) \Y, \theta_U^\top \theta_U \rangle \\
& + \langle (\nabla f(\U \Y^\top))^\top \U \rmD \W_\Y^{-1} [\theta_Y], \theta_Y \W_\Y \rangle + \langle \nabla f(\U \Y^\top) \Y \rmD \V_\Y^{-1} [\theta_Y], \theta_U \V_\Y \rangle\\
& + \langle \rmD \W_\Y\left[\overline{ \grad_\Y \, h_{r}([\U,\Y])}\right], \theta_Y^\top \theta_Y  \rangle /2 + \langle \rmD \V_\Y\left[\overline{ \grad_\Y \, h_{r}([\U,\Y])}\right], \theta_U^\top \theta_U  \rangle /2\\
& \overset{ \eqref{eq: FOSP-condition-general3} } = \nabla^2 f(\U \Y^\top)[\U \theta_Y^\top + \theta_U \Y^\top, \U \theta_Y^\top + \theta_U \Y^\top] + 2 \langle  \nabla f(\U \Y^\top), \theta_U \theta_Y^\top \rangle \\
& \overset{ \text{Proposition } \ref{prop: general-bijection3}, \eqref{eq: Hessian-con-gradient-general3} } =   \nabla^2 f(\X)[\cL^r_{\U,\Y}(\theta_{(\U,\Y)}),\cL^r_{\U,\Y}(\theta_{(\U,\Y)})] + 2 \langle \nabla f(\X), P_{\U_\perp} \theta_U \Y^\top \V \bSigma^{-1}  \theta_Y^\top  P_{\V_\perp}    \rangle\\
& = \Hess f(\X)[\cL^r_{\U,\Y}(\theta_{(\U,\Y)}),\cL^r_{\U,\Y}(\theta_{(\U,\Y)})],
\end{split}
\end{equation*}
where the last equality follows from the expression of $\Hess f(\X)$ in (<ref>) and the definition of $\cL^r_{\U,\Y}$.

Then, by (<ref>), (<ref>) and Theorem <ref>, we have $\overline{\Hess \, h_{r}([\U,\Y])}$ has $(p_1+p_2- r)r$ eigenvalues and $\widebar{\lambda}_i(\Hess \, h_{r}([\U,\Y]))$ is sandwiched between $(\sigma_r(\W^{-1}_\Y) \wedge \sigma_r^2(\Y \V_\Y^{-1/2} )) \lambda_i(\Hess f(\X)) $ and $( \sigma_1(\W_\Y^{-1}) \vee \sigma^2_1(\Y\V_\Y^{-1/2})   ) \lambda_i(\Hess f(\X)) $ for $i = 1,\ldots,(p_1+p_2- r)r$.  $\blacksquare$

§.§ Proof of Corollary <ref>.

Here we prove the Riemannian FOSP, SOSP, and strict saddle equivalence of (<ref>) on $\cM_{r}^e$ and $\cM_{r}^{q_1}$, similar proof applies to the equivalence on $\cM_{r}^e$ and $\cM_{r}^{q_2}$ (or $\cM_{r}^{q_3}$). First, by the connection of Riemannian gradients in (<ref>), the connection of Riemannian FOSPs under two geometries clearly holds.

Suppose $[\L,\R]$ is a Riemannian SOSP of (<ref>) and let $\X = \L \R^\top$. Given any  $\xi_\X \in T_{\X}\cM^e_{r}$, we have $\Hess f(\X)[\xi_\X, \xi_\X] \overset{ \eqref{eq: Hessian-connection-general1} }=\overline{\Hess \, h_{r}([\L,\R])}[(\cL^{r}_{\L,\R})^{-1}(\xi_\X), (\cL^{r}_{\L,\R})^{-1}(\xi_\X)] \geq 0$, where the inequality is by the SOSP assumption on $[\L,\R]$. Combining the fact $\X$ is a Riemannian FOSP, this shows $\X = \L \R^\top$ is a Riemannian SOSP under the embedded geometry.

Next, let us show the other direction: suppose $\X$ is a Riemannian SOSP under the embedded geometry, then there is a unique $[\L,\R]$ such that $\L \R^\top = \X$ and it is a Riemannian SOSP of (<ref>). To see this, first the uniqueness of $[\L,\R]$ is guaranteed by the fact $\bar{\ell}: (\L,\R) \in \widebar{\cM}_r^{q_1} \to \L \R^\top \in \cM_r^e$ induces a diffeomorphism between $\cM_r^{q_1}$ and $\cM_r^e$. In addition, we have shown $[\L,\R]$ is a Riemannian FOSP of (<ref>). Then by (<ref>), we have for any $\theta_{(\L,\R)} \in \cH_{(\L,\R)} \widebar{\cM}_{r}^{q_1}$, $\overline{\Hess \, h_{r}([\L,\R])}[\theta_{(\L,\R)}, \theta_{(\L,\R)}]  = \Hess f(\X)[\cL_{\L,\R}^{r}(\theta_{(\L,\R)}),\cL_{\L,\R}^{r}(\theta_{(\L,\R)})] \geq 0$.

Finally, the strict saddle equivalence follows from the sandwich inequality in Theorem <ref> and the definition of the strict saddle.

§ ADDITIONAL LEMMAS

The following lemma provides the connection of horizontal lifts of tangent vectors at different reference points for the quotient geometries we consider.

(i) Suppose $\theta_{[\Y]}$ is a tangent vector to the quotient manifold $\cM_{r+}^{q_1}$ at $[\Y]$. If $\W_\Y = \O \W_{\Y\O} \O^\top$ holds for any $\O \in \bbO_r$, then the horizontal lifts of $\theta_{[\Y]}$ at $\Y$ and $\Y\O$ for $\O \in \bbO_{r}$ are related as $\theta_{\Y \O} = \theta_{\Y} \O$.

(ii) Suppose $\theta_{[\U, \B]}$ is a tangent vector to the quotient manifold $\cM_{r+}^{q_2}$ at $[\U,\B]$. The horizontal lifts of $\theta_{[\U,\B]}$ at $(\U, \B)$ and at $(\U \O, \O^\top \B \O)$ are related as $\theta_{(\U \O, \O^\top \B \O)} =[(\theta_U \O)^\top \quad (\O^\top \theta_{B} \O)^\top ]^\top  $ given $\theta_{(\U,\B)} = [\theta_U^\top \quad \theta_B^\top]^\top$.

(iii) Suppose $\theta_{[\L, \R]}$ is a tangent vector to the quotient manifold $\cM_{r}^{q_1}$ at $[\L,\R]$. If $\W_{\L,\R} = \M \W_{\L\M,\R \M^{-\top}} \M^\top$ and $\V_{\L,\R} = \M^{-\top } \V_{\L\M,\R \M^{-\top}} \M^{-1}$ hold for any $\M \in \GL(r)$, then the horizontal lifts of $\theta_{[\L,\R]}$ at $(\L, \R)$ and at $(\L \M, \R \M^{-\top})$ are related as $\theta_{(\L \M, \R \M^{-\top})} =[(\theta_L \M)^\top \quad (\theta_{R} \M^{-\top})^\top ]^\top  $ given $\theta_{(\L,\R)} = [\theta_L^\top \quad \theta_R^\top]^\top$.

(iv) Suppose $\theta_{[\U, \B, \V]}$ is a tangent vector to the quotient manifold $\cM_{r}^{q_2}$ at $[\U,\B,\V]$. The horizontal lifts of $\theta_{[\U,\B,\V]}$ at $(\U, \B, \V)$ and at $(\U \O, \O^\top \B \O, \V\O)$ are related as $\theta_{(\U \O, \O^\top \B \O, \V \O)} =[(\theta_U \O)^\top \quad (\O^\top \theta_{B} \O)^\top \quad (\theta_V \O)^\top ]^\top  $ given $\theta_{(\U,\B,\V)} = [\theta_U^\top \quad \theta_B^\top \quad \theta_V^\top]^\top$.

(v) Suppose $\theta_{[\U, \Y]}$ is a tangent vector to the quotient manifold $\cM_{r}^{q_3}$ at $[\U,\Y]$. The horizontal lifts of $\theta_{[\U,\Y]}$ at $(\U, \Y)$ and at $(\U \O, \Y \O)$ are related as $\theta_{(\U \O, \Y \O)} =[(\theta_U \O)^\top \quad (\theta_{Y} \O)^\top ]^\top  $ given $\theta_{(\U,\Y)} = [\theta_U^\top \quad \theta_Y^\top]^\top$.

Proof of Lemma <ref>. Since the proofs for claims (ii)(iii)(iv)(v) are similar to the proof of the first claim, for simplicity, we only present the proof for the first one. Let $l: \cM_{r+}^{q_1} \to \bbR$ be an arbitrary smooth function and define $\bar{l}:= l\circ \pi: \widebar{\cM}^{q_1 }_{r+} \to \bbR$, where $\pi$ is the quotient mapping on the manifold $\cM_{r+}^{q_1}$. Let $z: \Y \to \Y \O$, so we have

\begin{equation} \label{eq: horizonal-lift-connect-eq1}
\bar{l}(\Y) = \bar{l}(z(\Y)), \quad \forall \Y \in \bbR^{p \times r}_*.
\end{equation}

By taking differential with respect to $\Y$ along direction $\theta_\Y \in \cH_\Y \widebar{\cM}_{r+}^{q_1}$ on both sides of (<ref>), we have

\begin{equation} \label{eq: horizonal-lift-connect-eq2}
\rmD \bar{l}(\Y)[\theta_\Y] = \rmD \bar{l}(z(\Y))[ \rmD z(\Y) [\theta_\Y] ].
\end{equation}
Moreover, by chain rule and the definition of the horizontal lift of a tangent vector, we have

\begin{equation} \label{eq: horizonal-lift-connect-eq3}
\rmD \bar{l}(\Y) [\theta_\Y] = \rmD l(\pi(\Y))[ \rmD \pi(\Y)[\theta_\Y]  ]  = \rmD l(\pi(\Y)) [ \theta_{[\Y]} ].
\end{equation}
In addition,

\begin{equation} \label{eq: horizonal-lift-connect-eq4}
\rmD z(\Y)[\theta_\Y] = \lim_{t \to 0} (z(\Y + t\theta_\Y) - z(\Y))/t = \theta_\Y \O.
\end{equation}

\begin{equation*}
\begin{split}
\rmD l(\pi(\Y\O)) [ \theta_{[\Y]} ] = \rmD l(\pi(\Y)) [ \theta_{[\Y]} ] \overset{\eqref{eq: horizonal-lift-connect-eq3}}= \rmD \bar{l}(\Y) [\theta_\Y]  &\overset{\eqref{eq: horizonal-lift-connect-eq2}}=  \rmD \bar{l}(z(\Y))[ \rmD z(\Y) [\theta_\Y]]\\
&\overset{\eqref{eq: horizonal-lift-connect-eq4}}=\rmD \bar{l}(z(\Y))[ \theta_\Y \O ] \\
&\overset{(a)}= \rmD l(\pi(\Y\O)) [ \rmD \pi(\Y \O)[\theta_{\Y} \O] ].
\end{split}
\end{equation*}
Here (a) is by chain rule and the definition of $z$. Since the above equation holds for any $l$, this implies $\rmD \pi(\Y \O)[\theta_{\Y} \O]= \theta_{[\Y]}$. Finally, by Lemma <ref>(i), we have $\cH_\Y \widebar{\cM}_{r+}^{q_1} = \{ \theta_\Y': \theta_\Y' = (\U \S + \U_\perp \D) \P^{-\top}, \D \in \bbR^{(p-r) \times r}, \S\P^{-\top}\W_\Y \P^{-1} \in \bbS^{r \times r} \}$, where $\U \in \st(r,p)$ spans the top $r$ eigenspace of $\Y\Y^\top$ and $\P = \U^\top \Y$. Then, it holds that $\cH_{\Y\O} \widebar{\cM}_{r+}^{q_1} = \{ \theta_{\Y\O}': \theta_{\Y\O}' = (\U \S + \U_\perp \D) \P^{-\top} \O, \D \in \bbR^{(p-r) \times r}, \S\P^{-\top}\W_\Y \P^{-1} \in \bbS^{r \times r} \}$. Thus, we have $\theta_\Y \O \in \cH_{\Y\O} \widebar{\cM}_{r+}^{q_1}$ given $\theta_\Y \in \cH_{\Y} \widebar{\cM}_{r+}^{q_1}$ and conclude $\theta_\Y \O$ is the unique horizontal lift of $\theta_{[\Y]}$ at $\Y \O$.

Suppose $f: \cM \to \bbR$ is a smooth function defined on the Riemannian quotient manifold $\cM = \widebar{\cM}/\sim$ and $\widebar{\cM}$ is endowed with the Riemannian metric $\bar{g}_\X$, which induces a Riemannian metric $g_{[\X]}$ on $\cM$. Then $\overline{\Hess f([\X])}[\theta_{[\X]}, \eta_{[\X]}] = \bar{g}_\X\left( \widebar{\nabla}_{\theta_\X} \overline{\grad f }, \eta_\X \right)$ for any $\theta_{[\X]}, \eta_{[\X]} \in T_{[\X]} \cM$.

Proof of Lemma <ref>. Suppose the vertical and horizontal spaces on $\widebar{\cM}$ are $\cV_\X \widebar{\cM}$ and $\cH_\X \widebar{\cM}$ with $\cV_\X \widebar{\cM} \oplus \cH_\X \widebar{\cM} = T_\X \widebar{\cM}$. Let us define $\cV'_\X \widebar{\cM}$ to be the subspace in $T_\X \widebar{\cM}$ that is orthogonal to $\cH_\X \widebar{\cM}$ with respect to $\bar{g}_\X$. In general, $\cV'_\X \widebar{\cM}$ is not equal to $\cV_\X \widebar{\cM}$ unless we pick the horizontal space in a canonical way.

Since $\widebar{\nabla}_{\theta_\X} \overline{\grad f }$ by definition belongs to $T_\X \widebar{\cM}$, we have for $\theta_\X, \eta_\X \in \cH_\X \widebar{\cM}$,

\begin{equation*}
\begin{split}
\overline{\Hess f([\X])}[\theta_{[\X]}, \eta_{[\X]}] &= \bar{g}_\X\left( \overline{\Hess f([\X])[\theta_{[\X]}]}, \eta_\X \right) \\
& \overset{\eqref{eq: quotient-hessian-linear-from} }=\bar{g}_\X\left( P_\X^{\cH}\left( \widebar{\nabla}_{\theta_\X} \overline{\grad f} \right),\eta_\X \right) \\
& = \bar{g}_\X\left( \widebar{\nabla}_{\theta_\X} \overline{\grad f}- P_\X^{\cV'}\left( \widebar{\nabla}_{\theta_\X} \overline{\grad f} \right),\eta_\X \right)\\
& \overset{(a)}= \bar{g}_\X\left( \widebar{\nabla}_{\theta_\X} \overline{\grad f }, \eta_\X \right),
\end{split}
\end{equation*}
where (a) is because $\eta_\X \in \cH_\X \widebar{\cM}$ and $P_\X^{\cH}$ and $P_\X^{\cV'}$ denote the projection operators onto $\cH_\X \widebar{\cM}$ and $\cV'_\X \widebar{\cM}$, respectively.   $\blacksquare$

(Max-min Theorem for Eigenvalues <cit.> ) Suppose $\A$ is a Hermitian operator on the Hilbert space $\cH$ with dimension $p$ and inner product $g(\cdot,\cdot)$ and $\A$ has eigenvalues $\lambda_1 \geq \lambda_2 \geq \cdots \geq \lambda_p$. If $\scC_k$ denotes the set of $k$-dimensional subspaces of $\cH$, then $\lambda_k = \max_{C \in \scC_k} \min_{\u \in C, \u \neq 0 } g(\u, \A \u)/g(\u,\u) .$