s_{k}}{\partial[\widetilde{W}_{l}]_{ij}}\right]\right)\widetilde{W}_{k}^{\top}W_{k+1}^{\top}\ldots\Psi_{L}(s_{L})\widetilde{W}_{L}^{\top}$
$\displaystyle+W_{1}^{\top}\Psi_{1}(s_{1})\ldots\Psi_{l}(s_{l})\delta_{ij}^{m_{l}\times
n_{l}}W_{l+1}^{\top}\Psi_{l+1}(s_{l+1})\ldots\Psi_{L}(s_{L})\widetilde{W}_{L}^{\top}$
$\displaystyle=\;$
$\displaystyle\sum_{k=l+1}^{L}W_{1}^{\top}\ldots\Psi_{k}^{\prime}(s_{k})\textup{
diag}\left(\left[W_{k}\widetilde{W}_{k-1}\Psi_{k-1}(s_{k-1})\ldots
W_{l+1}\mathrm{e}_{i}^{(n_{l})}[\sigma_{l}(s_{l})]_{j}\right]\right)\widetilde{W}_{k}^{\top}\ldots\widetilde{W}_{L}^{\top}$
$\displaystyle+W_{1}^{\top}\Psi_{1}(s_{1})\ldots\Psi_{l}(s_{l})\delta_{ij}^{m_{l}\times
n_{l}}W_{l+1}^{\top}\Psi_{l+1}(s_{l+1})\ldots\Psi_{L}(s_{L})\widetilde{W}_{L}^{\top}$

and

$\frac{\partial^{2}\widehat{\Phi}}{\partial[\widetilde{W}_{L}]_{1j}\partial
x}=W_{1}^{\top}\Psi_{1}(s_{1})\widetilde{W}_{1}^{\top}\ldots
W_{L}^{\top}\Psi_{L}(s_{L})\delta_{1j}^{m_{l}\times 1}$

for $j\in\\{1,\ldots,m_{L}\\}$ as $\widetilde{W}_{L}\in\mathbb{R}^{1\times
m_{L}}$. All calculated derivatives are columns of the matrix
$\frac{\partial^{2}\widehat{\Phi}}{\partial v\partial x}\in\mathbb{R}^{n\times
p}$. To apply Lemma 3.23 we show that the matrix
$\frac{\partial^{2}\widehat{\Phi}}{\partial v\partial x}\in\mathbb{R}^{n\times
p}$ has at any point
$(x,v)\in\mathcal{X}\times\left(\textup{vecm}(\mathbb{W}^{\ast})\times\textup{stk}(\mathbb{B})\right)$
full rank, i.e. rank $n$ as $n\leq p$. As elementary column operations do not
change the rank of a matrix, we can subtract for fixed $l\in\\{1,\ldots,L\\}$
from the columns
$\frac{\partial^{2}\widehat{\Phi}}{\partial[W_{l}]_{ij}\partial x}$,
$i\in\\{1,\ldots,m_{l}\\}$, $j\in\\{1,\ldots,n_{l-1}\\}$, corresponding to the
weight matrix $W_{l}$, the columns
$\frac{\partial^{2}\widehat{\Phi}}{\partial[b_{l}]_{i}\partial x}$,
$i\in\\{1,\ldots,m_{l}\\}$, corresponding to the bias $b_{l}$, each multiplied
with $[h_{l}]_{j}$, $j\in\\{1,\ldots,n_{l-1}\\}$. We obtain

$\frac{\partial^{2}\widehat{\Phi}}{\partial[W_{l}]_{ij}\partial
x}-\frac{\partial^{2}\widehat{\Phi}}{\partial[b_{l}]_{i}\partial
x}\cdot[h_{l}]_{j}=W_{1}^{\top}\Psi_{1}(s_{1})\ldots\Psi_{l-1}(s_{l-1})\widetilde{W}_{l-1}^{\top}\delta_{ij}^{n_{l-1}\times
m_{l}}\Psi_{l}(s_{l})\ldots\Psi_{L}(s_{L})\widetilde{W}_{L}^{\top}.$ (3.2)

Analogously we can without changing the rank of the matrix subtract for fixed
$l\in\\{1,\ldots,L\\}$ from the columns
$\frac{\partial^{2}\widehat{\Phi}}{\partial[\widetilde{W}_{l}]_{ij}\partial
x}$, $i\in\\{1,\ldots,n_{l}\\}$, $j\in\\{1,\ldots,m_{l}\\}$, corresponding to
the weight matrix $\widetilde{W}_{l}$, the columns
$\frac{\partial^{2}\widehat{\Phi}}{\partial[\tilde{b}_{l}]_{i}\partial x}$,
$i\in\\{1,\ldots,n_{l}\\}$, corresponding to the bias $\tilde{b}_{l}$, each
multiplied with every $[\sigma_{l}(s_{l})]_{j}$, $j\in\\{1,\ldots,m_{l}\\}$.
We obtain

$\frac{\partial^{2}\widehat{\Phi}}{\partial[\widetilde{W}_{l}]_{ij}\partial
x}-\frac{\partial^{2}\widehat{\Phi}}{\partial[\tilde{b}_{l}]_{i}\partial
x}\cdot[\sigma_{l}(s_{l})]_{j}=W_{1}^{\top}\Psi_{1}(s_{1})\ldots\Psi_{l}(s_{l})\delta_{ij}^{m_{l}\times
n_{l}}W_{l+1}^{\top}\Psi_{l+1}(s_{l+1})\ldots\Psi_{L}(s_{L})\widetilde{W}_{L}^{\top}.$
(3.3)

As the neural network $\Phi$ is augmented, there exists a layer $g_{l^{\ast}}$
of maximal width, $l^{\ast}\in\\{1,\ldots,2L-1\\}$, which has at least $n+1$
nodes, i.e. $d_{l^{\ast}}\geq n+1$. If $l^{\ast}$ is even, the layer
$g_{l^{\ast}}$ of maximal width corresponds to the layer $h_{l^{\ast}/2}$, and
we consider the modified columns (3.2) for the index $l^{\ast}/2+1$. We obtain

$\displaystyle\;\frac{\partial^{2}\widehat{\Phi}}{\partial[W_{l^{\ast}/2+1}]_{ij}\partial
x}-\frac{\partial^{2}\widehat{\Phi}}{\partial[b_{l^{\ast}/2+1}]_{i}\partial
x}\cdot[h_{l^{\ast}/2+1}]_{j}$ $\displaystyle=\,$
$\displaystyle\;W_{1}^{\top}\ldots\widetilde{W}_{l^{\ast}/2}^{\top}\delta_{ij}^{n_{l^{\ast}/2}\times
m_{l^{\ast}/2+1}}\Psi_{l^{\ast}/2+1}(s_{l^{\ast}/2+1})\ldots\widetilde{W}_{L}^{\top}$
(3.4) $\displaystyle\eqqcolon$
$\displaystyle\;Y\delta_{ij}^{n_{l^{\ast}/2}\times{m_{l^{\ast}/2+1}}}Z$
$\displaystyle=$ $\displaystyle\;Y_{i}\cdot[Z]_{j}\in\mathbb{R}^{n},$ (3.5)

where $Y=[Y_{1},\ldots,Y_{n_{l^{\ast}/2}}]\in\mathbb{R}^{n\times
n_{l^{\ast}/2}}$ and $Z\in\mathbb{R}^{m_{l^{\ast}/2+1}}$ are implicitly
defined by equation (3.4). Expression (3.5) follows from the simple structure
of the matrix $\delta_{ij}^{n_{l^{\ast}/2}\times m_{l^{\ast}/2+1}}$: it holds

$Y\delta_{ij}^{n_{l^{\ast}/2}\times
m_{l^{\ast}/2+1}}Z=Y\mathrm{e}_{i}^{(n_{l^{\ast}/2})}[Z]_{j}=Y_{i}\cdot[Z]_{j},$

where $\mathrm{e}_{i}^{(n_{l^{\ast}/2})}$ denotes the $i$-th unit vector in
$\mathbb{R}^{n_{l^{\ast}/2}}$ and $[Z]_{j}$ the $j$-th component of the vector
$Z$. If $l^{\ast}$ is odd, the layer $g_{l^{\ast}}$ of maximal width
corresponds to the layer $s_{(l^{\ast}+1)/2}$, and we consider the modified
columns (3.2) for the index $(l^{\ast}+1)/2$. Analogously we obtain

$\displaystyle\frac{\partial^{2}\widehat{\Phi}}{\partial[\widetilde{W}_{(l^{\ast}+1)/2}]_{ij}\partial
x}-\frac{\partial^{2}\widehat{\Phi}}{\partial[\tilde{b}_{(l^{\ast}+1)/2}]_{i}\partial
x}\cdot[\sigma_{(l^{\ast}+1)/2}(s_{(l^{\ast}+1)/2})]_{j}$ $\displaystyle=\,$
$\displaystyle\;W_{1}^{\top}\ldots\Psi_{(l^{\ast}+1)/2}(s_{(l^{\ast}+1)/2})\delta_{ij}^{m_{(l^{\ast}+1)/2}\times
n_{(l^{\ast}+1)/2}}W_{(l^{\ast}+3)/2}^{\top}\ldots\widetilde{W}_{L}^{\top}$
(3.6) $\displaystyle\eqqcolon$
$\displaystyle\;\widetilde{Y}\delta_{ij}^{m_{(l^{\ast}+1)/2}\times
n_{(l^{\ast}+1)/2}}\widetilde{Z}$ $\displaystyle=$
$\displaystyle\;\widetilde{Y}_{i}\cdot[\widetilde{Z}]_{j}\in\mathbb{R}^{n},$
(3.7)

where
$\widetilde{Y}=[\widetilde{Y}_{1},\ldots,\widetilde{Y}_{m_{(l^{\ast}+1)/2}}]\in\mathbb{R}^{n\times
m_{(l^{\ast}+1)/2}}$ and $\widetilde{Z}\in\mathbb{R}^{n_{(l^{\ast}+1)/2}}$ are
implicitly defined by equation (3.6). If $(l^{\ast}+1)/2=L$, the matrix
product $\widetilde{Z}$ does not exist, so we define
$\widetilde{Z}=1\in\mathbb{R}^{n_{L}}=\mathbb{R}$. In both cases we obtain
from equations (3.5) or (3.7) modified columns of the form

$\overline{Y}_{i}[\overline{Z}]_{j}\in\mathbb{R}^{n},\qquad
i\in\\{1,\ldots,d_{l^{\ast}}\\},j\in\\{1,\ldots,d_{l^{\ast}+1}\\}$

with
$\overline{Y}=[\overline{Y}_{1},\ldots,\overline{Y}_{d_{l^{\ast}}}]\in\mathbb{R}^{n\times
d_{l^{\ast}}}$ and $\overline{Z}\in\mathbb{R}^{d_{l^{\ast}+1}}$., where
$d_{l}^{\ast}\geq n+1$ is the number of nodes in the layer of maximal width
$g_{l^{\ast}}$.

In the following we aim to show that for every
$(x,v)\in\mathcal{X}\times\left(\textup{vecm}(\mathbb{W}^{\ast})\times\textup{stk}(\mathbb{B})\right)$,
there exists an index $j\in\\{1,\ldots,d_{l^{\ast}+1}\\}$, such that the
matrix

$Q_{j}\coloneqq\left[\overline{Y}_{1}\cdot[\overline{Z}]_{j},\ldots,\overline{Y}_{d_{l^{\ast}}}\cdot[\overline{Z}]_{j}\right]\in\mathbb{R}^{n\times
d_{l^{\ast}}}$ (3.8)

has full rank $n$. As $Q_{j}$ is a submatrix of
$\frac{\partial^{2}\widehat{\Phi}}{\partial v\partial x}\in\mathbb{R}^{n\times
p}$ of rank $n$, also the matrix $\frac{\partial^{2}\widehat{\Phi}}{\partial
v\partial x}$ has full rank $n$ for every
$(x,v)\in\mathcal{X}\times\left(\textup{vecm}(\mathbb{W}^{\ast})\times\textup{stk}(\mathbb{B})\right)$.
Due to the assumption
$(W_{1},\widetilde{W}_{1},\ldots,W_{L},\widetilde{W}_{L})\in\mathbb{W}^{\ast}$
all considered weight matrices as well as the diagonal matrices
$\Psi_{l}(s_{l})$, $l\in\\{1,\ldots,L\\}$ have full rank. Since the neural
network $\Phi$ is augmented, it holds $d_{i}\leq d_{j}$ for $0\leq i<j\leq
l^{\ast}$ and $d_{i}\geq d_{j}$ for $l^{\ast}\leq i<j\leq L$. By Lemma 3.10,
also the matrix products $\overline{Y}$ and $\overline{Z}$ have full rank,
i.e. $\textup{rank}(\overline{Y})=n$ and $\textup{rank}(\overline{Z})=1$ (this
also holds in the special case where $\overline{Z}=1$ for $(l^{\ast}+1)/2=L$).
Consequently the vector $\overline{Z}\in\mathbb{R}^{d_{l^{\ast}+1}}$ has at
least one non-zero entry, i.e. $[\overline{Z}]_{j^{\ast}}\neq 0$ for some
$j^{\ast}\in\\{1,\ldots,d_{l^{\ast}+1}\\}$. As $\overline{Z}$ depends on
$(x,v)\in\mathcal{X}\times\left(\textup{vecm}(\mathbb{W}^{\ast})\times\textup{stk}(\mathbb{B})\right)$,
also the choice of $j^{\ast}$ depends on $x$ and $v$. For the index
$j^{\ast}$, the matrix $Q_{j^{\ast}}$ arises from the matrix $\overline{Y}$ by
elementary column operations, as each column is multiplied by the non-zero
scalar $[Z]_{j^{\ast}}$. As the matrix $\overline{Y}$ has full rank $n$, also
the matrix $Q_{j^{\ast}}$ has full rank $n$, which implies that
$\frac{\partial^{2}\widehat{\Phi}}{\partial v\partial x}$ has full rank $n$
for every
$(x,v)\in\mathcal{X}\times\left(\textup{vecm}(\mathbb{W}^{\ast})\times\textup{stk}(\mathbb{B})\right)$.
As the map $\widehat{\Phi}\in
C^{k}\left(\mathcal{X}\times\left(\textup{vecm}(\mathbb{W}^{\ast})\times\textup{stk}(\mathbb{B})\right)\right)$,
$k\geq 2$, fulfills the assumptions of Lemma 3.23, it follows that for all
weights
$v\in\mathcal{V}\coloneqq\textup{vecm}(\mathbb{W}^{\ast})\times\textup{stk}(\mathbb{B})$,
except possibly for a zero set $\mathcal{V}_{0}\subset\mathcal{V}$ w.r.t. the
Lebesgue measure in $\mathbb{R}^{p}$, the corresponding augmented classical
feed-forward neural network
$\Phi\in\Xi^{k}_{\textup{A},\mathbb{W}^{\ast}}(\mathcal{X},\mathbb{R})$,
$k\geq 2$ with weight vector $v$ is a Morse function.

As
$\textup{vecm}(\mathbb{W}\setminus\mathbb{W}^{\ast})\times\textup{stk}(\mathbb{B})=\mathbb{R}^{p}\setminus\mathcal{V}$
is by Lemma 3.6 a zero set w.r.t. the Lebesgue measure in $\mathbb{R}^{p}$,
also the finite union
$(\mathbb{R}^{p}\setminus\mathcal{V})\cup\mathcal{V}_{0}$ is a zero set w.r.t.
the Lebesgue measure in $\mathbb{R}^{p}$. It follows that for all weights
$v\in\textup{vecm}(\mathbb{W})\times\textup{stk}(\mathbb{B})=\mathbb{R}^{p}$,
except possibly for a zero set w.r.t. the Lebesgue measure in
$\mathbb{R}^{p}$, the augmented classical feed-forward neural network
$\Phi\in\Xi^{k}_{\textup{A}}(\mathcal{X},\mathbb{R})$, $k\geq 2$ with weight
vector $v$ is a Morse function. The Morse function $\Phi$ can be of class
$(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$ or of class
$(\mathcal{C}2)^{k}(\mathcal{X},\mathbb{R})$, as by Lemma 3.21 it is possible
that the augmented network $\Phi$ can or cannot have critical points for
$W\in\mathbb{W}^{\ast}$. ∎

######  Remark 3.26.

The proof of Theorem 3.25 relies on the fact, that the matrix $Q_{j^{\ast}}$
defined in (3.8) has full rank as $\overline{Y}$ and $\overline{Z}$ are full
rank matrices. This can be guaranteed, as the considered neural network is
augmented and the weight matrices are assumed to have full rank. As this is
the only time that the augmented structure of $\Phi$ is used, Theorem 3.25 is
also applicable for neural networks with a bottleneck, where $\overline{Y}$
and $\overline{Z}$ are full rank matrices, see Section 3.5.

We end this section with an example of a one-layer augmented neural network
illustrating the assertions of Theorems 3.22 and 3.25.

######  Example 3.27.

Consider the one-layer neural network architecture
$\Phi\in\Xi^{\infty}_{\textup{A}}(\mathbb{R},\mathbb{R})$ with $L=1$ defined
by the weights and biases

$W_{1}=\begin{pmatrix}w_{1}\\\ w_{2}\end{pmatrix}\in\mathbb{R}^{2},\qquad
b_{1}=\begin{pmatrix}b_{11}\\\
b_{12}\end{pmatrix}\in\mathbb{R}^{2},\qquad\widetilde{W}_{1}=(\widetilde{w}_{1},\widetilde{w}_{2})\in\mathbb{R}^{1\times
2},\qquad\tilde{b}_{1}\in\mathbb{R},$

and soft-plus activation functions
$[\sigma_{1}]_{1}(x)=[\sigma_{1}]_{2}(x)=\ln(1+\exp(x))$. $\Phi$ is visualized
in Figure 3.7. By Theorem 3.25, the network $\Phi$ is for all
$v=(w_{1},w_{2},\widetilde{w}_{1},\widetilde{w}_{2},b_{11},b_{12},\tilde{b}_{1})^{\top}\in\mathbb{R}^{7}$,
except possibly for a zero set w.r.t. the Lebesgue measure in $\mathbb{R}^{7}$
a Morse function, which we verify in the following. By Lemma 3.17 it holds for
the gradient

$\displaystyle\nabla\Phi(x)$
$\displaystyle=(w_{1},w_{2})\begin{pmatrix}[\sigma_{1}]_{1}^{\prime}([s_{1}]_{1})&0\\\
0&[\sigma_{1}]_{2}^{\prime}([s_{1}]_{2})\end{pmatrix}\begin{pmatrix}\widetilde{w}_{1}\\\
\widetilde{w}_{2}\end{pmatrix}$
$\displaystyle=\frac{w_{1}\widetilde{w}_{1}}{1+\exp(-s_{1})}+\frac{w_{2}\widetilde{w}_{2}}{1+\exp(-s_{2})}$

and the Hessian is given by

$H_{\Phi}(x)=\frac{w_{1}^{2}\widetilde{w}_{1}\exp(-s_{1})}{(1+\exp(-s_{1}))^{2}}+\frac{w_{2}^{2}\widetilde{w}_{2}\exp(-s_{2})}{(1+\exp(-s_{2}))^{2}},$

where $s_{1}=w_{1}x+b_{11}$ and $s_{2}=w_{2}x+b_{12}$. The set of weights

$\mathcal{W}_{0}\coloneqq\left\\{v\in\mathbb{R}^{7}:w_{1}=0\vee
w_{2}=0\vee\widetilde{w}_{1}=0\vee\widetilde{w}_{2}=0\vee
w_{1}=w_{2}\right\\}$

describes a lower-dimensional set and hence is a zero set w.r.t. the Lebesgue
measure in $\mathbb{R}^{7}$. From now on we assume
$v\in\mathbb{R}^{7}\setminus\mathcal{W}_{0}$. If $\Phi$ has no critical point
it is of class $(\mathcal{C}1)^{\infty}(\mathbb{R},\mathbb{R})$. Otherwise let
$x^{\ast}\in\mathbb{R}$ be a critical point, then the Hessian matrix evaluated
at $x^{\ast}$ is given by

$H_{\Phi}(x^{\ast})=\frac{\widetilde{w}_{1}}{\widetilde{w}_{2}}\frac{w_{1}^{2}}{(1+\exp(-s_{1}^{\ast}))^{2}}\left(\widetilde{w}_{2}\exp(-s_{1}^{\ast})+\widetilde{w}_{1}\exp(-s_{2}^{\ast})\right),$

where we used the algebraic constraint $\nabla\Phi(x^{\ast})=0$ and defined
$s_{1}^{\ast}=w_{1}x^{\ast}+b_{11}$ and $s_{2}^{\ast}=w_{2}x^{\ast}+b_{12}$.
As $v\in\mathbb{R}^{7}\setminus\mathcal{W}_{0}$ it holds

$H_{\Phi}(x^{\ast})=0\qquad\Leftrightarrow\qquad
g(v,x^{\ast})\coloneqq\widetilde{w}_{2}\exp(-s_{1}^{\ast})+\widetilde{w}_{1}\exp(-s_{2}^{\ast})=0.$

If $\widetilde{w}_{1}\widetilde{w}_{2}>0$, $g(v,x^{\ast})$ is always non-zero,
such that every critical point is non-degenerate. Otherwise
$\widetilde{w}_{1}\widetilde{w}_{2}<0$ and the equation $g(v,x^{\ast})=0$ is
for fixed $v$ uniquely solvable for $x^{\ast}$:

$g(v,x^{\ast})=0\qquad\Leftrightarrow\qquad
x^{\ast}=h(v)=\frac{\ln(-\widetilde{w}_{2}/\widetilde{w}_{1})-b_{11}+b_{12}}{w_{1}-w_{2}}.$
(3.9)

This implies, that for fixed $v\in\mathbb{R}^{7}\setminus\mathcal{W}_{0}$,
there can exist at most one degenerate critical point. It is possible that
that multiple critical points exist, but (3.9) implies that there cannot be
multiple degenerate critical points. For the upcoming analysis, we have to
exclude the set

$\mathcal{W}_{1}\coloneqq\\{v\in\mathbb{R}^{7}\setminus\mathcal{W}_{0}:1+\exp(-w_{1}h(v)-b_{11})+w_{1}h(v)=0\\}$

from the weight space. $\mathcal{W}_{1}$ has Lebesgue measure zero in
$\mathbb{R}^{7}$ as for $\varepsilon>0$, it holds for the weight

$\hat{v}(v,\varepsilon)=v+(0,0,0,0,\varepsilon,\varepsilon,0)=(w_{1},w_{2},\tilde{w}_{1},\tilde{w}_{2},b_{11}+\varepsilon,b_{12}+\varepsilon,\tilde{b}_{1})$

that $h(\hat{v})=h(v)$, which implies that if $v\in\mathcal{W}_{1}$, then
$\hat{v}\notin\mathcal{W}_{1}$ as $\exp(-\varepsilon)\neq 1$, such that the
constraint defining $\mathcal{W}_{1}$ cannot be fulfilled. As $\varepsilon>0$
was arbitrary, $\mathcal{W}_{1}$ cannot have non-zero Lebesgue measure.

Figure 3.7: One-layer augmented neural network architecture
$\Phi\in\Xi^{\infty}_{\textup{A}}(\mathbb{R},\mathbb{R})$ of Example 3.27.

In the following we show, that the set of weights

$\mathcal{W}_{2}\coloneqq\\{v\in\mathbb{R}^{7}\setminus(\mathcal{W}_{0}\cup\mathcal{W}_{1}):\;\exists\textup{
critical point }x^{\ast}\in\mathbb{R}\textup{ with }g(v,x^{\ast})=0\\},$

such that $\Phi$ has a degenerate critical point, has Lebesgue measure zero.
To that purpose assume there exist weights
$v\in\mathbb{R}^{7}\setminus(\mathcal{W}_{0}\cup\mathcal{W}_{1})$, such that
$\Phi$ has a degenerate critical point $x^{\ast}$. We notice that by Theorem
3.22 a non-degenerate critical point perturbs to a non-degenerate critical
point under small perturbation of the weights. Hence, if we aim to find
degenerate critical points in a neighborhood of the considered weights
$v\in\mathbb{R}^{7}\setminus(\mathcal{W}_{0}\cup\mathcal{W}_{1})$, the only
possibility is that the degenerate critical point $x^{\ast}$ perturbs to a
degenerate critical point. For $\varepsilon>0$ sufficiently small, consider
the modified weights

$\bar{v}(v,\varepsilon)=v+\left(\varepsilon,0,\delta(v,\varepsilon),0,0,0,0\right)^{\top}=\left(w_{1}+\varepsilon,w_{2},\widetilde{w}_{1}+\delta(v,\varepsilon),\widetilde{w}_{2},b_{11},b_{12},\tilde{b}_{1}\right)^{\top}\in\mathbb{R}^{7},$

where

$\delta(v,\varepsilon)=\widetilde{w}_{1}\left(\frac{w_{1}}{w_{1}+\varepsilon}\frac{1+\exp(-(w_{1}+\varepsilon)x^{\ast}-b_{11})}{1+\exp(-s_{1}^{\ast})}-1\right),$
(3.10)

such that

$\frac{w_{1}\widetilde{w}_{1}}{1+\exp(-s_{1}^{\ast})}=\frac{(w_{1}+\varepsilon)(\widetilde{w}_{1}+\delta(v,\varepsilon))}{1+\exp(-(w_{1}+\varepsilon)x^{\ast}-b_{11})}.$

Hence, if and only if $x^{\ast}$ is a critical point of the network with
weights $v$, then $x^{\ast}$ is also a critical point of the network with
weights $\bar{v}$. It holds

$\displaystyle\frac{\partial
g(\bar{v}(v,\varepsilon),x^{\ast})}{\partial\varepsilon}$
$\displaystyle=-\widetilde{w}_{2}x^{\ast}\exp(-(w_{1}+\varepsilon)x^{\ast}-b_{11})+\frac{\partial\delta(v,\varepsilon)}{\partial\varepsilon}\exp(-s_{2}^{\ast})$
$\displaystyle=-\widetilde{w}_{2}x^{\ast}\exp(-(w_{1}+\varepsilon)x^{\ast}-b_{11})+\left(-\frac{\widetilde{w}_{1}w_{1}}{(w_{1}+\varepsilon)^{2}}\frac{1+\exp(-(w_{1}+\varepsilon)x^{\ast}-b_{11})}{1+\exp(-s_{1}^{\ast})}\right.$
$\displaystyle\hskip
11.38109pt\left.-\frac{\widetilde{w}_{1}w_{1}x^{\ast}}{w_{1}+\varepsilon}\frac{\exp(-(w_{1}+\varepsilon)x^{\ast}-b_{11})}{1+\exp(-s_{1}^{\ast})}\right)\exp(-s_{2}^{\ast}),$

which yields evaluated at $\varepsilon=0$:

$\displaystyle\frac{\partial g(\bar{v}(v,0),x^{\ast})}{\partial\varepsilon}$
$\displaystyle=-\widetilde{w}_{2}x^{\ast}\exp(-s_{1}^{\ast})-\frac{\widetilde{w}_{1}}{w_{1}}\exp(-s_{2}^{\ast})-\widetilde{w}_{1}x^{\ast}\frac{\exp(-s_{1}^{\ast})\exp(-s_{2}^{\ast})}{1+\exp(-s_{1}^{\ast})}$
$\displaystyle=-x^{\ast}(\widetilde{w}_{2}\exp(-s_{1}^{\ast})+\widetilde{w}_{1}\exp(-s_{2}^{\ast}))-\frac{\widetilde{w}_{1}}{w_{1}}\exp(-s_{2}^{\ast})+\widetilde{w}_{1}x^{\ast}\frac{\exp(-s_{2}^{\ast})}{1+\exp(-s_{1}^{\ast})}$
$\displaystyle=\frac{\widetilde{w}_{1}\exp(-s_{2}^{\ast})}{w_{1}(1+\exp(-s_{1}^{\ast}))}\left(1+\exp(-w_{1}h(v)-b_{11})+w_{1}h(v)\right)$

where we used in the last line the property $g(v,x^{\ast})=0$, such that the
first summand vanishes, and we replaced $x^{\ast}$ by $h(v)$ in the second
summand. As $v\in\mathbb{R}^{7}\setminus(\mathcal{W}_{0}\cup\mathcal{W}_{1})$,
it holds

$\frac{\partial g(\bar{v}(v,0),x^{\ast})}{\partial\varepsilon}\neq 0.$ (3.11)

For $\varepsilon>0$ sufficiently small we can deduce from $g(v,x^{\ast})=0$
and (3.11) that $g(\bar{v}(v,\varepsilon),x^{\ast})\neq 0$, such that
$x^{\ast}$ is a non-degenerate critical point for the modified weights
$\bar{v}$. From (3.10) it follows that for every $\mu>0$, the parameter
$\varepsilon>0$ can be chosen small enough, such that
$|\delta(v,\varepsilon)|<\mu$. Hence, we found a point $\bar{v}$ in a
$\mu$-neighborhood of $v\in\mathcal{W}_{2}$, which is not contained in
$\mathcal{W}_{2}$. Consequently $\mathcal{W}_{2}$ must have zero Lebesgue
measure in $\mathbb{R}^{7}$ and $\Phi$ is a Morse function for all weighs in
$\mathbb{R}^{7}\setminus(\mathcal{W}_{0}\cup\mathcal{W}_{1}\cup\mathcal{W}_{2})$,
where $\mathcal{W}_{0}\cup\mathcal{W}_{1}\cup\mathcal{W}_{2}$ is a set of
Lebesgue measure zero in $\mathbb{R}^{7}$.

###  3.5 Analysis of Bottleneck Architectures

In the last section, we proved Theorem 3.25 and showed regularity of the
critical points of augmented neural networks. As mentioned in Remark 3.26, the
theorem can also be applied to neural networks with a bottleneck, if they have
a specific structure. In the following we study different types of neural
networks with a bottleneck, and show that they can be of class
$(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$ ,
$(\mathcal{C}2)^{k}(\mathcal{X},\mathbb{R})$ or
$(\mathcal{C}3)^{k}(\mathcal{X},\mathbb{R})$. The main difference between the
types of bottlenecks defined in the following theorem are if the neural
network has an augmented part and if the assumptions specified in Remark 3.26
are fulfilled.

######  Theorem 3.28.

Let $\Phi\in\Xi_{\textup{B}}^{k}(\mathcal{X},\mathbb{R})$, $k\geq 2$,
$\mathcal{X}\subset\mathbb{R}^{n}$ be a classical feed-forward neural network
with weight matrices
$W=(W_{1},\widetilde{W}_{1},\ldots,W_{L},\widetilde{W}_{L})=(V_{1},\ldots,V_{2L})\in\mathbb{W}$.
Assume that $\Phi$ has its first bottleneck in layer
$g_{j^{\ast}}\in\mathbb{R}^{d_{j^{\ast}}}$. Define for
$l\in\\{1,\ldots,2L-1\\}$ the matrix product

$Z_{l}(x)\coloneqq\widetilde{W}_{L}\Psi_{L}(s_{L})W_{L}\ldots
V_{l+1}\in\mathbb{R}^{1\times d_{l}},$

where $V_{l+1}=W_{l/2+1}$ if $l$ is even and $V_{l+1}=\widetilde{W}_{(l+1)/2}$
if $l$ is odd. $Z_{l}(x)$ is the product of all matrices occurring in the
gradient defined in Lemma 3.17 until $V_{l+1}$.

  1. (a)

Let the first bottleneck of $\Phi$ be non-augmented, i.e.
$n=d_{0}\geq\ldots>d_{j^{\ast}}$ and let $Z_{j^{\ast}}(x)\neq 0$ for all
$x\in\mathcal{X}$. If $W\in\mathbb{W}^{\ast}$, $\Phi$ is of class
$(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$. Consequently for all set of
weights $(\textup{vecm}(W);\textup{stk}(B))\in\mathbb{R}^{N_{W}+N_{B}}$,
$W\in\mathbb{W}$, $b\in\mathbb{B}$, except possibly for a zero set in
$\mathbb{R}^{N_{W}+N_{B}}$ w.r.t. the Lebesgue measure, $\Phi$ is of class
$(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$.

  2. (b)

Let the first bottleneck of $\Phi$ be non-augmented, i.e.
$n=d_{0}\geq\ldots>d_{j^{\ast}}$ and denote the set of points
$x\in\mathcal{X}$ such that $Z_{j^{\ast}}(x)=0$ by
$\mathcal{X}^{\ast}\neq\varnothing$. Then every critical point
$x^{\ast}\in\mathcal{X}^{\ast}$ is degenerate, such that $\Phi$ is of class
$(\mathcal{C}3)^{k}(\mathcal{X},\mathbb{R})$.

  3. (c)

Let the first bottleneck of $\Phi$ be augmented, i.e. there exists an index
$i^{\ast}<j^{\ast}$ with $n=d_{0}\leq\ldots<d_{i^{\ast}}$ and let
$Z_{i^{\ast}}(x)\neq 0$ for all $x\in\mathcal{X}$. Then $\Phi$ is for all
weights $(\textup{vecm}(W);\textup{stk}(B))\in\mathbb{R}^{N_{W}+N_{B}}$,
$W\in\mathbb{W}^{\ast}$, $b\in\mathbb{B}$, except possibly for a zero set in
$\mathbb{R}^{N_{W}+N_{B}}$ w.r.t. the Lebesgue measure, of class
$(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$ or
$(\mathcal{C}2)^{k}(\mathcal{X},\mathbb{R})$. The same statement holds if
$\mathbb{W}$ is replaced by $\mathbb{W}^{\ast}$.

  4. (d)

Let the first bottleneck of $\Phi$ be augmented, i.e. there exists an index
$i^{\ast}<j^{\ast}$ with $n=d_{0}\leq\ldots<d_{i^{\ast}}$ and and denote the
set of points $x\in\mathcal{X}$ such that $Z_{i^{\ast}}(x)=0$ by
$\mathcal{X}^{\ast}\neq\varnothing$. Then $x^{\ast}\in\mathcal{X}^{\ast}$ can
be either degenerate or non-degenerate, so $\Phi$ is of class
$(\mathcal{C}2)^{k}(\mathcal{X},\mathbb{R})$ or
$(\mathcal{C}3)^{k}(\mathcal{X},\mathbb{R})$.

###### Proof.

Let $\Phi\in\Xi^{k}_{\textup{B}}(\mathcal{X},\mathbb{R})$, with weight
matrices
$W=(W_{1},\widetilde{W}_{1},\ldots,W_{L},\widetilde{W}_{L})=(V_{1},\ldots,V_{2L})\in\mathbb{W}$.
By definition, there exist layers $g_{i}^{\ast}$, $g_{j}^{\ast}$ and
$g_{l}^{\ast}$ with $0\leq i^{\ast}<j^{\ast}<l^{\ast}\leq 2L$, such that
$d_{i}^{\ast}>d_{j}^{\ast}$ and $d_{j}^{\ast}<d_{l}^{\ast}$. The network
gradient is by Lemma 3.17 given by

$\displaystyle\nabla\Phi(x)^{\top}$
$\displaystyle=\widetilde{W}_{L}\Psi_{L}(s_{L})W_{L}\ldots
V_{l+1}V_{l}\ldots\widetilde{W}_{1}\Psi_{1}(s_{1})W_{1}$
$\displaystyle=V_{2L}\Psi_{L}(s_{L})V_{2L-1}\ldots V_{l+1}V_{l}\ldots
V_{2}\Psi_{1}(s_{1})V_{1}$

with $l\in\\{1,\ldots,2L-1\\}$, where
$V_{l}=\widetilde{W}_{l/2}\in\mathbb{R}^{d_{l}\times d_{l-1}}$,
$V_{l+1}=W_{l/2+1}\in\mathbb{R}^{d_{l+1}\times d_{l}}$ if $l$ is even and
$V_{l}=\Psi_{(l+1)/2}(s_{(l+1)/2})\in\mathbb{R}^{d_{l}\times d_{l}}$,
$V_{l+1}=\widetilde{W}_{(l+1)/2}\in\mathbb{R}^{d_{l+1}\times d_{l}}$ if $l$ is
odd. We denote by $Z_{l}(x)\in\mathbb{R}^{1\times d_{l}}$ the matrix product
$\widetilde{W}_{L}\ldots V_{l+1}$ and by $Y_{l}(x)\in\mathbb{R}^{d_{l}\times
n}$ the matrix product $V_{l}\ldots W_{1}$, such that

$\nabla\Phi(x)^{\top}=Z_{l}(x)Y_{l}(x)\in\mathbb{R}^{1\times n}.$ (3.12)

For case (a), we assume that $n=d_{0}\geq\ldots>d_{j^{\ast}}$ and that
$Z_{j^{\ast}}(x)\neq 0$ for all $x\in\mathcal{X}$. If $W\in\mathbb{W}^{\ast}$,
the assumption $n=d_{0}\geq\ldots>d_{j^{\ast}}$ implies that the dimensions of
the matrices in the product $Y_{j^{\ast}}(x)$ are monotonically increasing,
such that by Lemma 3.10 $Y_{j^{\ast}}(x)$ has for every $x\in\mathcal{X}$ full
rank $d_{j^{\ast}}$. As $Z_{j^{\ast}}(x)\neq 0$ for all $x\in\mathcal{X}$ also
the matrix $Z_{j^{\ast}}(x)$ has always full rank 1, such that another
application of Lemma 3.10 implies that $\nabla\Phi(x)$ has for every
$x\in\mathcal{X}$ full rank. Consequently no critical point exists and $\Phi$
is of class $(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$ if
$W\in\mathbb{W}^{\ast}$. The same argumentation as in Theorem 3.18 for non-
augmented neural networks shows that for all set of weights
$(\textup{vecm}(W);\textup{stk}(B))\in\mathbb{R}^{N_{W}+N_{B}}$, except
possibly for a zero set in $\mathbb{R}^{N_{W}+N_{B}}$ w.r.t. the Lebesgue
measure, $\Phi$ is of class $(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$.

For case (b), we assume that $n=d_{0}\geq\ldots>d_{j^{\ast}}$ and that
$Z_{j^{\ast}}(x^{\ast})=0$ for all
$x^{\ast}\in\mathcal{X}^{\ast}\neq\varnothing$. The structure of the gradient
(3.12) implies that every $x^{\ast}\in\mathcal{X}^{\ast}$ is a critical point
of $\Phi$. The $i$-th row $[H_{\Phi}(x)^{\top}]_{i}^{\top}$ of the Hessian
$H_{\Phi}(x)\in\mathbb{R}^{n\times n}$ evaluated at a critical point
$x^{\ast}\in\mathcal{X}^{\ast}$ is given by

$[H_{\Phi}(x^{\ast})^{\top}]_{i}^{\top}=\frac{\partial}{\partial
x_{i}}\left[Z_{j^{\ast}}(x)Y_{j^{\ast}}(x)\right]\bigg{|}_{x=x^{\ast}}=\frac{\partial
Z_{j^{\ast}}}{\partial
x_{i}}(x^{\ast})Y_{j^{\ast}}(x^{\ast})+Z_{j^{\ast}}(x^{\ast})\frac{\partial
Y_{j^{\ast}}}{\partial x_{i}}(x^{\ast})=\frac{\partial Z_{j^{\ast}}}{\partial
x_{i}}(x^{\ast})Y_{j^{\ast}}(x^{\ast}),$

as $Z_{j^{\ast}}(x^{\ast})=0$ for $x^{\ast}\in\mathcal{X}^{\ast}$. The $n$
vectors $\frac{\partial Z_{j^{\ast}}}{\partial
x_{i}}(x^{\ast})\in\mathbb{R}^{1\times d_{j^{\ast}}}$, $i\in\\{1,\ldots,n\\}$
are linearly dependent as $d_{j^{\ast}}<n$. Hence, there exist
$\alpha_{1},\ldots,\alpha_{n}$ not all equal to zero such that

$\sum_{i=1}^{n}\alpha_{i}\frac{\partial Z_{j^{\ast}}}{\partial
x_{i}}(x^{\ast})=0\in\mathbb{R}^{1\times d_{j^{\ast}}}.$

As matrix multiplication is distributive, it follows

$\sum_{i=1}^{n}\alpha_{i}[H_{\Phi}(x^{\ast})^{\top}]_{i}^{\top}=\sum_{i=1}^{n}\alpha_{i}\left[\frac{\partial
Z_{j^{\ast}}}{\partial
x_{i}}(x^{\ast})Y_{j^{\ast}}(x^{\ast})\right]=\left[\sum_{i=1}^{n}\alpha_{i}\frac{\partial
Z_{j^{\ast}}}{\partial
x_{i}}(x^{\ast})\right]Y_{j^{\ast}}(x^{\ast})=0\in\mathbb{R}^{n},$

which implies that the rows of the Hessian $H_{\Phi}(x^{\ast})$ are linearly
dependent. Consequently $H_{\Phi}(x^{\ast})$ has not full rank and at least
one eigenvalue is zero, such that the critical point $x^{\ast}$ is degenerate.
As $x^{\ast}$ was arbitrary, it follows that every critical point of $\Phi$ is
degenerate and $\Phi$ is of class
$(\mathcal{C}3)^{k}(\mathcal{X},\mathbb{R})$.

For case (c) we assume that there exists an index $i^{\ast}<j^{\ast}$ with
$n=d_{0}\leq\ldots<d_{i^{\ast}}$ and that $Z_{i^{\ast}}(x)\neq 0$ for all
$x\in\mathcal{X}$. We aim to show the statement of Theorem 3.25 for the
considered bottleneck architecture, as proposed in Remark 3.26. To that
purpose, we compare the matrices $\overline{Y}$ and $\overline{Z}$ of the
proof of Theorem 3.25 with the matrices $Y_{i^{\ast}}(x)$ and
$Z_{i^{\ast}}(x)$ defined in this proof by identifying the index $l^{\ast}$ of
Theorem 3.25 with the index $i^{\ast}$ considered here. If $i^{\ast}$ is even,
equation (3.4) implies that $Y_{i^{\ast}}(x)=\overline{Y}^{\top}$ and
$Z_{i^{\ast}}(x)=\overline{Z}^{\top}W_{i^{\ast}/2+1}$, and if $i^{\ast}$ is
odd, (3.6) shows that $Y_{i^{\ast}}(x)=\overline{Y}^{\top}$ and
$Z_{i^{\ast}}(x)=\overline{Z}^{\top}\widetilde{W}_{(i^{\ast}+1)2}$. If
$W\in\mathbb{W}^{\ast}$, the assumption $n=d_{0}\leq\ldots<d_{j^{\ast}}$
implies that the dimensions of the matrices in the product $Y_{i^{\ast}}(x)$
are monotonically decreasing, such that by Lemma 3.10 $Y_{i^{\ast}}(x)$ has
for every $x\in\mathcal{X}$ full rank $n$, and hence also $\overline{Y}$ has
full rank $n$. The assumption $Z_{i^{\ast}}(x)\neq 0$ for all
$x\in\mathcal{X}$ implies that it is necessary that $\overline{Z}\neq 0$ for
all $x\in\mathcal{X}$, so $\overline{Z}$ has full rank 1. By Remark 3.26, the
proof of Theorem 3.25 works analogously for the considered neural network
$\Phi$ with a bottleneck, such that $\Phi$ is for all weights
$(\textup{vecm}(W);\textup{stk}(B))\in\mathbb{R}^{N_{W}+N_{B}}$,
$W\in\mathbb{W}^{\ast}$, $b\in\mathbb{B}$, except possibly for a zero set in
$\mathbb{R}^{N_{W}+N_{B}}$ w.r.t. the Lebesgue measure, of class
$(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$ or
$(\mathcal{C}2)^{k}(\mathcal{X},\mathbb{R})$. As
$\textup{vecm}(\mathbb{W}\setminus\mathbb{W}^{\ast})\times\textup{stk}(\mathbb{B})$
is a zero set w.r.t the Lebesgue measure in $\mathbb{R}^{N_{W}+N_{B}}$ and the
union of two sets with measure zero has again measure zero, the same statement
holds if $\mathbb{W}^{\ast}$ is replaced by $\mathbb{W}$.

For case (d) we assume that there exists an index $i^{\ast}<j^{\ast}$ with
$n=d_{0}\leq\ldots<d_{i^{\ast}}$ and that $Z_{i^{\ast}}(x^{\ast})=0$ for all
$x^{\ast}\in\mathcal{X}^{\ast}\neq\varnothing$. The structure of the gradient
(3.12) implies that every $x^{\ast}\in\mathcal{X}^{\ast}$ is a critical point
of $\Phi$. As in case (b), the $i$-th row $[H_{\Phi}(x)^{\top}]_{i}^{\top}$ of
the Hessian $H_{\Phi}(x)\in\mathbb{R}^{n\times n}$ evaluated at a critical
point $x^{\ast}\in\mathcal{X}^{\ast}$ is given by

$[H_{\Phi}(x^{\ast})^{\top}]_{i}^{\top}=\frac{\partial}{\partial
x_{i}}\left[Z_{j^{\ast}}(x)Y_{j^{\ast}}(x)\right]\bigg{|}_{x=x^{\ast}}=\frac{\partial
Z_{j^{\ast}}}{\partial
x_{i}}(x^{\ast})Y_{j^{\ast}}(x^{\ast})+Z_{j^{\ast}}(x^{\ast})\frac{\partial
Y_{j^{\ast}}}{\partial x_{i}}(x^{\ast})=\frac{\partial Z_{j^{\ast}}}{\partial
x_{i}}(x^{\ast})Y_{j^{\ast}}(x^{\ast}),$

as $Z_{j^{\ast}}(x^{\ast})=0$ for $x^{\ast}\in\mathcal{X}^{\ast}$. As
$d_{j^{\ast}}>n$, the $n$ vectors $\frac{\partial Z_{j^{\ast}}}{\partial
x_{i}}(x^{\ast})\in\mathbb{R}^{1\times d_{j^{\ast}}}$, $i\in\\{1,\ldots,n\\}$
can be linearly dependent, but they can also be linearly independent. This
implies in analogy to case (b), that the Hessian matrix can have full rank,
but does not need to. In Example 3.29 we show that both cases are possible. As
$\Phi$ always has at least one critical point, it follows that $\Phi$ is of
class $(\mathcal{C}2)^{k}(\mathcal{X},\mathbb{R})$ or
$(\mathcal{C}3)^{k}(\mathcal{X},\mathbb{R})$. ∎

We end this section with a few examples of neural networks with bottleneck
architectures, which show that all cases mentioned in Theorem 3.28 exist. The
considered architectures are visualized in Figure 3.8.

######  Example 3.29.

We present for each of the cases (a)-(d) of Theorem 3.28 a two-layer neural
network architecture $\Phi\in\Xi_{\textup{B}}^{k}(\mathcal{X},\mathbb{R})$,
$k\geq 2$ with full rank weight matrices $W\in\mathbb{W}^{\ast}$ and
nonlinear, strictly monotonically increasing activation functions.

  1. (a)

Let $\mathcal{X}=\mathbb{R}^{2}$, $L=2$ and $\Phi$ be defined by the weight
matrices

$W_{1}=(1,1),\quad\widetilde{W}_{1}=\begin{pmatrix}1\\\ 1\end{pmatrix},\quad
W_{2}=(1,1),\quad\widetilde{W}_{2}=1$

and arbitrary biases $b\in\mathbb{B}$, see Figure 3.88(a). Then $\Phi$ has a
non-augmented bottleneck at layer $g_{1}=s_{1}$, so $j^{\ast}=1$. It holds

$Z_{1}(x)=1\cdot\Psi_{2}(s_{2})\cdot(1,1)\cdot\begin{pmatrix}1\\\
1\end{pmatrix}=2\sigma_{2}^{\prime}(s_{2})\neq
0\qquad\forall\,x\in\mathcal{X}.$

By Theorem 3.28(a), $\Phi$ is off class
$(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$. As

$\nabla\Phi(x)^{\top}=1\cdot\Psi_{2}(s_{2})\cdot(1,1)\cdot\begin{pmatrix}1\\\
1\end{pmatrix}\cdot\Psi_{1}(s_{1})\cdot(1,1)=(2\sigma_{2}^{\prime}(s_{2})\sigma_{1}^{\prime}(s_{1}),2\sigma_{2}^{\prime}(s_{2})\sigma_{1}^{\prime}(s_{1}))\neq(0,0)$

for all $x\in\mathcal{X}$, we verified that $\Phi$ cannot have any critical
points.

  2. (b)

Let $\mathcal{X}=\mathbb{R}^{2}$, $L=2$ and $\Phi$ be defined by the weight
matrices

$W_{1}=(1,1),\quad\widetilde{W}_{1}=\begin{pmatrix}1\\\ 0\end{pmatrix},\quad
W_{2}=(0,1),\quad\widetilde{W}_{2}=1$

and arbitrary biases $b\in\mathbb{B}$, see Figure 3.88(b). Then $\Phi$ has a
non-augmented bottleneck at layer $g_{1}=s_{1}$, so $j^{\ast}=1$. It holds

$Z_{1}(x)=1\cdot\Psi_{2}(s_{2})\cdot(0,1)\cdot\begin{pmatrix}1\\\
0\end{pmatrix}=0\qquad\forall\,x\in\mathcal{X}.$

By Theorem 3.28(b), $\Phi$ is off class
$(\mathcal{C}3)^{k}(\mathcal{X},\mathbb{R})$. As

$\nabla\Phi(x)^{\top}=1\cdot\Psi_{2}(s_{2})\cdot(0,1)\cdot\begin{pmatrix}1\\\
0\end{pmatrix}\cdot\Psi_{1}(s_{1})\cdot(1,1)=(2\sigma_{2}^{\prime}(s_{2})\sigma_{1}^{\prime}(s_{1}),2\sigma_{2}^{\prime}(s_{2})\sigma_{1}^{\prime}(s_{1}))=(0,0)$

for all $x\in\mathcal{X}$, the gradient $\nabla\Phi$ is the constant zero
function, hence the Hessian matrix $H_{\Phi}$ is also zero everywhere. Hence,
we verified that $\Phi$ is of class
$(\mathcal{C}3)^{k}(\mathcal{X},\mathbb{R})$, as every critical point is
degenerate.

.

(a) Example of a neural network
$\Phi\in\Xi_{\textup{B}}^{k}(\mathbb{R}^{2},\mathbb{R})$ with a non-augmented
bottleneck at layer $s_{1}$, which is of class
$(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$.

.

(b) Example of a neural network
$\Phi\in\Xi_{\textup{B}}^{k}(\mathbb{R}^{2},\mathbb{R})$ with a non-augmented
bottleneck at layer $s_{1}$, which is of class
$(\mathcal{C}3)^{k}(\mathcal{X},\mathbb{R})$.

.

.

(c) Example of a neural network
$\Phi\in\Xi_{\textup{B}}^{k}(\mathbb{R},\mathbb{R})$ with an augmented
bottleneck at layer $h_{1}$, which is depending on $\alpha$ of class
$(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$ or
$(\mathcal{C}2)^{k}(\mathcal{X},\mathbb{R})$.

.

(d) Example of a neural network
$\Phi\in\Xi_{\textup{B}}^{k}(\mathbb{R},\mathbb{R})$ with an augmented
bottleneck at layer $h_{1}$, which is depending on $\alpha$ and $\beta$ of
class $(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$,
$(\mathcal{C}2)^{k}(\mathcal{X},\mathbb{R})$ or
$(\mathcal{C}3)^{k}(\mathcal{X},\mathbb{R})$

.

Figure 3.8: Two-layer neural network architectures
$\Phi\in\Xi_{\textup{B}}^{k}(\mathcal{X},\mathbb{R})$ with full rank weight
matrices $W\in\mathbb{W}^{\ast}$ and nonlinear, strictly monotonically
increasing activation functions, which are analyzed in Example 3.29.

  3. (c)

Let $\mathcal{X}=\mathbb{R}$, $L=2$ and $\Phi$ be defined by the weight
matrices

$W_{1}=\begin{pmatrix}1\\\
1\end{pmatrix},\quad\widetilde{W}_{1}=(1,\alpha),\quad
W_{2}=\begin{pmatrix}1\\\ 1\end{pmatrix},\quad\widetilde{W}_{2}=(1,1)$

with $\alpha\in\mathbb{R}$ specified in the following and arbitrary biases
$b\in\mathbb{B}$, see Figure 3.88(c). Then $\Phi$ has an augmented bottleneck
at layer $g_{2}=h_{1}$, so $i^{\ast}=1$ and $j^{\ast}=2$. It holds

$\displaystyle Z_{1}(x)$
$\displaystyle=(1,1)\cdot\Psi_{2}(s_{2})\cdot\begin{pmatrix}1\\\
1\end{pmatrix}\cdot(1,\alpha)$
$\displaystyle=\big{(}[\sigma_{2}]_{1}^{\prime}([s_{2}]_{1})+[\sigma_{2}]_{2}^{\prime}([s_{2}]_{2}),\alpha([\sigma_{2}]_{1}^{\prime}([s_{2}]_{1})+[\sigma_{2}]_{2}^{\prime}([s_{2}]_{2}))\big{)}\neq
0\qquad\forall\,x\in\mathcal{X},$

as the sum of two strictly monotonically increasing functions is again
strictly monotonically increasing. By Theorem 3.28(c), $\Phi$ is for all set
of weights, except possibly for a zero set w.r.t. the Lebesgue measure, of
class $(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$ or
$(\mathcal{C}2)^{k}(\mathcal{X},\mathbb{R})$. We show that both cases are
possible. It holds

$\displaystyle\nabla\Phi(x)^{\top}$
$\displaystyle=(1,1)\cdot\Psi_{2}(s_{2})\cdot\begin{pmatrix}1\\\
1\end{pmatrix}\cdot(1,\alpha)\cdot\Psi_{1}(s_{1})\cdot\begin{pmatrix}1\\\
1\end{pmatrix}$
$\displaystyle=\big{(}[\sigma_{2}]_{1}^{\prime}([s_{2}]_{1})+[\sigma_{2}]_{2}^{\prime}([s_{2}]_{2})\big{)}\cdot\big{(}[\sigma_{1}]_{1}^{\prime}([s_{1}]_{1})+\alpha[\sigma_{1}]_{2}^{\prime}([s_{1}]_{2})\big{)}.$

We define the set

$\mathcal{A}=\left\\{a\in\mathbb{R}:a=-\frac{[\sigma_{1}]_{1}^{\prime}([s_{1}]_{1})}{[\sigma_{1}]_{2}^{\prime}([s_{1}]_{2})}\textup{
for some }x\in\mathbb{R}\right\\}\subset\mathbb{R}_{<0},$

which is non-empty and by construction of the network independent of the
choice of $\alpha$. Also $\mathbb{R}\setminus\mathcal{A}$ is non-empty as
$\mathcal{A}\subset\mathbb{R}_{<0}$. If
$\alpha\in\mathbb{R}\setminus\mathcal{A}$, then $\nabla\Phi(x)\neq 0$ for all
$x\in\mathcal{X}$, such that $\Phi$ is of class
$(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$. If the activation functions
$[\sigma_{1}]_{1}$ and $[\sigma_{1}]_{2}$ are non-linear, then
$[\sigma_{1}]_{1}^{\prime}$ and $[\sigma_{1}]_{2}^{\prime}$ are non-constant,
such that $\mathcal{A}$ has non-zero Lebesgue measure. Hence, for all
$\alpha\in\mathcal{A}$, except possibly for a zero set w.r.t. the Lebesgue
measure, Theorem 3.28(c) implies that $\Phi$ is of class
$(\mathcal{C}2)^{k}(\mathcal{X},\mathbb{R})$ and has only non-degenerate
critical points.

  4. (d)

Let $\mathcal{X}=\mathbb{R}$, $L=2$ and $\Phi$ be defined by the weight
matrices

$W_{1}=\begin{pmatrix}1\\\
1\end{pmatrix},\quad\widetilde{W}_{1}=(1,\alpha),\quad
W_{2}=\begin{pmatrix}1\\\ 1\end{pmatrix},\quad\widetilde{W}_{2}=(1,\beta)$

with $\alpha,\beta\in\mathbb{R}$ specified in the following and arbitrary
biases $b\in\mathbb{B}$, see Figure 3.88(d). It holds

$\displaystyle Z_{1}(x)$
$\displaystyle=(1,\beta)\cdot\Psi_{2}(s_{2})\cdot\begin{pmatrix}1\\\
1\end{pmatrix}\cdot(1,\alpha)$
$\displaystyle=\big{(}[\sigma_{2}]_{1}^{\prime}([s_{2}]_{1})+\beta[\sigma_{2}]_{2}^{\prime}([s_{2}]_{2}),\alpha([\sigma_{2}]_{1}^{\prime}([s_{2}]_{1})+\beta[\sigma_{2}]_{2}^{\prime}([s_{2}]_{2}))\big{)}.$

We define the set

$\mathcal{B}_{\alpha}=\left\\{b\in\mathbb{R}:b=-\frac{[\sigma_{2}]_{1}^{\prime}([s_{2}]_{1})}{[\sigma_{2}]_{2}^{\prime}([s_{2}]_{2})}\textup{
for some }x\in\mathbb{R}\right\\},$

which is non-empty and by construction of the network dependent on the choice
of $\alpha$, but independent of the choice of $\beta$. If
$\beta\notin\mathcal{B}_{\alpha}$, the analysis of the neural network is the
same as in case (c) and $\Phi$ can be of class
$(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$ or
$(\mathcal{C}2)^{k}(\mathcal{X},\mathbb{R})$. In the following we choose
$\beta\in\mathcal{B}_{\alpha}$, such that for every choice of $\alpha,\beta$
there exist $x^{\ast}\in\mathbb{R}$ such that $Z_{1}(x^{\ast})=0$ and the
assumptions of Theorem 3.28(d) are fulfilled. Consequently $\Phi$ is off class
$(\mathcal{C}2)^{k}(\mathcal{X},\mathbb{R})$ or
$(\mathcal{C}3)^{k}(\mathcal{X},\mathbb{R})$. We show that both cases are
possible. It holds

$\displaystyle\nabla\Phi(x)^{\top}$
$\displaystyle=(1,\beta)\cdot\Psi_{2}(s_{2})\cdot\begin{pmatrix}1\\\
1\end{pmatrix}\cdot(1,\alpha)\cdot\Psi_{1}(s_{1})\cdot\begin{pmatrix}1\\\
1\end{pmatrix}$
$\displaystyle=\big{(}[\sigma_{2}]_{1}^{\prime}([s_{2}]_{1})+\beta[\sigma_{2}]_{2}^{\prime}([s_{2}]_{2})\big{)}\cdot\big{(}[\sigma_{1}]_{1}^{\prime}([s_{1}]_{1})+\alpha[\sigma_{1}]_{2}^{\prime}([s_{1}]_{2})\big{)}.$

As $\beta\in\mathcal{B}_{\alpha}$, for every choice of $\alpha,\beta$ there
exist at least one critical point $x^{\ast}\in\mathbb{R}$. The Hessian matrix
$H_{\Phi}(x)$ is by the product rule given by

$\displaystyle H_{\Phi}(x)$
$\displaystyle=\left([\sigma_{2}]_{1}^{\prime\prime}([s_{2}]_{1})\frac{\partial[s_{2}]_{1}}{\partial
x}+\beta[\sigma_{2}]_{2}^{\prime\prime}([s_{2}]_{2})\frac{\partial[s_{2}]_{2}}{\partial
x}\right)\cdot\left([\sigma_{1}]_{1}^{\prime}([s_{1}]_{1})+\alpha[\sigma_{1}]_{2}^{\prime}([s_{1}]_{2})\right)$
$\displaystyle\;\;\;+\left([\sigma_{2}]_{1}^{\prime}([s_{2}]_{1})+\beta[\sigma_{2}]_{2}^{\prime}([s_{2}]_{2})\right)\cdot\left([\sigma_{1}]_{1}^{\prime\prime}([s_{1}]_{1})\frac{\partial[s_{1}]_{1}}{\partial
x}+\alpha[\sigma_{1}]_{2}^{\prime\prime}([s_{1}]_{2})\frac{\partial[s_{1}]_{2}}{\partial
x}\right).$

Evaluated at a critical point $x^{\ast}$ it holds

$H_{\Phi}(x^{\ast})=\left([\sigma_{2}]_{1}^{\prime\prime}([s_{2}]_{1})\frac{\partial[s_{2}]_{1}}{\partial
x}+\beta[\sigma_{2}]_{2}^{\prime\prime}([s_{2}]_{2})\frac{\partial[s_{2}]_{2}}{\partial
x}\right)\cdot\left([\sigma_{1}]_{1}^{\prime}([s_{1}]_{1})+\alpha[\sigma_{1}]_{2}^{\prime}([s_{1}]_{2})\right)$

as $Z_{1}(x^{\ast})=0$. By choosing $\alpha\in\mathcal{A}$, which is defined
in part (c), we can guarantee that the critical point $x^{\ast}$ can be
degenerate, so it is possible that $\Phi$ is of class
$(\mathcal{C}3)^{k}(\mathcal{X},\mathbb{R})$. To show that $\Phi$ can also be
of class $(\mathcal{C}2)^{k}(\mathcal{X},\mathbb{R})$, we choose some
$\alpha\in\mathbb{R}\setminus\mathcal{A}$, such that
$[\sigma_{1}]_{1}^{\prime}([s_{1}]_{1})+\alpha[\sigma_{1}]_{2}^{\prime}([s_{1}]_{2})\neq
0$ for all $x\in\mathbb{R}$. $\alpha\in\mathbb{R}\setminus\mathcal{A}$ exists
by part (c). If we choose for example soft-plus activation functions
$[\sigma_{2}]_{1}(x)=[\sigma_{2}]_{2}(x)=\ln(1+\exp(x))$, then

$\frac{\partial[s_{2}]_{1}}{\partial x}=\frac{\partial[s_{2}]_{2}}{\partial
x}=[\sigma_{1}]_{1}^{\prime}([s_{1}]_{1})+\alpha[\sigma_{1}]_{2}^{\prime}([s_{1}]_{2}),$

such that

$\displaystyle H_{\Phi}(x^{\ast})$
$\displaystyle=\left(\frac{\exp(-[s_{2}]_{1})}{(1+\exp(-[s_{2}]_{1}))^{2}}+\beta\frac{\exp(-[s_{2}]_{2})}{(1+\exp(-[s_{2}]_{2}))^{2}}\right)\cdot\left([\sigma_{1}]_{1}^{\prime}([s_{1}]_{1})+\alpha[\sigma_{1}]_{2}^{\prime}([s_{1}]_{2})\right)^{2}$
$\displaystyle=\left(\frac{\exp(-[s_{2}]_{1})}{(1+\exp(-[s_{2}]_{1}))^{2}}-\frac{1+\exp(-[s_{2}]_{2})}{1+\exp(-[s_{2}]_{1})}\frac{\exp(-[s_{2}]_{2})}{(1+\exp(-[s_{2}]_{2}))^{2}}\right)$
$\displaystyle\hskip
11.38109pt\cdot\left([\sigma_{1}]_{1}^{\prime}([s_{1}]_{1})+\alpha[\sigma_{1}]_{2}^{\prime}([s_{1}]_{2})\right)^{2}$
$\displaystyle=\frac{1}{1+\exp(-[s_{2}]_{1})}\left(\frac{1}{1+\exp([s_{2}]_{1})}-\frac{1}{1+\exp([s_{2}]_{2})}\right)\cdot\left([\sigma_{1}]_{1}^{\prime}([s_{1}]_{1})+\alpha[\sigma_{1}]_{2}^{\prime}([s_{1}]_{2})\right)^{2}$

which is non-zero as long as $[s_{2}]_{1}\neq[s_{2}]_{2}$ as
$\frac{1}{1+\exp(x)}$ is a monotonically decreasing function. As
$s_{2}=W_{1}h_{1}+b_{2}$, variation of the bias $b_{2}\in\mathbb{R}^{2}$
guarantees that for all $b_{2}\in\mathbb{R}^{2}$, except possibly for a set of
measure w.r.t. the Lebesgue measure in $\mathbb{R}^{2}$, it holds
$[s_{2}]_{1}\neq[s_{2}]_{2}$. Hence, we showed the existence of a neural
network architecture $\Phi$ which fulfills the assumptions of Theorem 3.28(d)
and which is of class $(\mathcal{C}2)^{k}(\mathcal{X},\mathbb{R})$.

##  4 Neural ODEs

Each neural ODE architecture is based on the solution
$h:\mathcal{I}\rightarrow\mathbb{R}^{m}$ of an initial value problem

$\frac{\mathrm{d}h}{\mathrm{d}t}=f(h(t),t),\qquad
h(0)=s\in\mathcal{S}\subset\mathbb{R}^{m},$ (IVP)

with $f:\mathbb{R}^{m}\times\mathcal{I}\rightarrow\mathbb{R}^{m}$, where
$\mathcal{I}\subset\mathbb{R}$ open denotes the maximal time interval of
existence. We denote a solution with initial condition $h(0)=s$ by $h_{s}(t)$
to take into account the dependence on the initial data. Hereby
$\varnothing\neq\mathcal{S}\subset\mathbb{R}^{m}$ is a non-zero set of initial
conditions. The considered neural ODE architectures depend on the time-$T$ map
$h_{s}(T)$ of (IVP), hence we always assume that the maximal time interval of
existence fulfills $\mathcal{I}\supset[0,T]$. The vector field
$f:\mathbb{R}^{m}\times\mathcal{I}\rightarrow\mathbb{R}^{m}$ is a continuous
function, which can depend on parameters. As the results established in this
work do not depend on the choice of the vector field, we consider no specific
paramterization of $f$. First, we state a basic result from ODE theory
regarding the regularity of the solution map of the initial value problem
(IVP).

######  Lemma 4.1 ([8]).

Let $f\in C^{k,0}(\mathbb{R}^{m}\times\mathcal{I},\mathbb{R}^{m})$, $k\geq 0$
and assume $[0,T]\subset\mathcal{I}$. Then it holds for the time-$T$ map of
the solution $h_{s}:\mathcal{I}\rightarrow\mathbb{R}^{m}$ of the initial value
problem (IVP) that $h_{s}(T)\in C^{k}(\mathcal{S},\mathbb{R}^{m})$ and if
$k\geq 1$, the solution curves are unique.

We need to assume the regularity of the vector field, to be able to
differentiate the network and characterize if the neural ODE is a Morse
function. The analysis is restricted to neural ODE architectures with scalar
output, for more general networks, the results apply to the scalar components
of the output. The architectures we consider have a linear layer before and
after the time-$T$ map of the initial value problem resulting in the neural
ODE architecture

$\Phi:\mathcal{X}\mapsto\mathbb{R},\qquad\Phi(x)\coloneqq\widetilde{L}(h_{L(x)}(T))=\widetilde{W}\cdot
h_{Wx+b}(T)+\tilde{b}=\widetilde{W}\cdot h_{s}(T)+\tilde{b},$ (4.1)

where the first linear layer $L:\mathbb{R}^{n}\rightarrow\mathbb{R}^{m}$ is
represented by a matrix $W\in\mathbb{R}^{m\times n}$ and a vector
$b\in\mathbb{R}^{m}$ and the second linear layer
$\widetilde{L}:\mathbb{R}^{m}\rightarrow\mathbb{R}$ is represented by a row
vector $\widetilde{W}\in\mathbb{R}^{1\times m}$ and a scalar
$\tilde{b}\in\mathbb{R}$. The intermediate layer, which defines the initial
condition of (IVP) is calculated by $s=L(x)=Wx+b$. We define the set of
initial conditions of (IVP) to be $\mathcal{S}\coloneqq L(\mathcal{X})$.
Compared to classical feed-forward neural networks, the neural ODE
architecture (4.1) has one inner linear layer corresponding to the weights
$W_{1}$ and $b_{1}$, then an initial value problem evolves in the dimension
$m$ of the intermediate layer $s\coloneqq Wx+b$ and then a second outer linear
layer corresponding to the weights $\widetilde{W}_{L}$ and $\tilde{b}_{L}$ is
applied to the time-$T$ map of (IVP). The regularity of the neural ODE
architecture (4.1) follows directly from the regularity of the underlying
initial value problem.

######  Lemma 4.2.

Let $f\in C^{k,0}(\mathbb{R}^{m}\times\mathcal{I},\mathbb{R}^{m})$, $k\geq 0$
and assume $[0,T]\subset\mathcal{I}$. Then it holds for the neural ODE
architecture defined in (4.1) that $\Phi\in C^{k}(\mathcal{X},\mathbb{R})$.

###### Proof.

The result follows directly from Lemma 4.1. ∎

######  Definition 4.3.

For $k\geq 0$, the set of all scalar neural ODE architectures
$\Phi:\mathcal{X}\rightarrow\mathbb{R}$, $\mathcal{X}\subset\mathbb{R}^{n}$
open, defined by (4.1) with vector field $f\in
C^{k,0}(\mathbb{R}^{m}\times\mathcal{I},\mathbb{R}^{m})$ and
$[0,T]\subset\mathcal{I}$, $\mathcal{I}\subset\mathbb{R}$ open, is denoted by
$\textup{NODE}^{k}(\mathcal{X},\mathbb{R})\subset
C^{k}(\mathcal{X},\mathbb{R})$.

###  4.1 Special Architectures

In the following, we subdivide the set of all neural ODE architectures
$\text{NODE}^{k}(\mathcal{X},\mathbb{R})$ introduced in Definition 4.3 in
three different classes: non-augmented neural ODEs, augmented neural ODEs and
neural ODEs with a bottleneck.

#### Non-Augmented

We call a neural ODE $\Phi\in\text{NODE}^{k}(\mathcal{X},\mathbb{R})$ non-
augmented, if $n\geq m$, i.e. the dimension of the ODE is smaller than the
dimension of the input data. We require that the weight matrices of both
linear layers have full rank, i.e. $\text{rank}(W)=m$ and
$\text{rank}(\widetilde{W})=1$. The subset of non-augmented neural ODEs is
denoted by

$\textup{NODE}^{k}_{\textup{N}}(\mathcal{X},\mathbb{R})\coloneqq\left\\{\Phi\in\textup{NODE}^{k}(\mathcal{X},\mathbb{R}):\Phi\textup{
is non-augmented}\right\\}.$

#### Augmented

We call a neural ODE $\Phi\in\text{NODE}^{k}(\mathcal{X},\mathbb{R})$
augmented, if $n<m$, i.e. the dimension of the ODE is larger than the
dimension of the input data. We require that the weight matrices of both
linear layers have full rank, i.e. $\text{rank}(W)=n$ and
$\text{rank}(\widetilde{W})=1$. The subset of augmented neural ODEs is denoted
by

$\textup{NODE}^{k}_{\textup{A}}(\mathcal{X},\mathbb{R})\coloneqq\left\\{\Phi\in\textup{NODE}^{k}(\mathcal{X},\mathbb{R}):\Phi\textup{
is augmented}\right\\}.$

#### Bottleneck

We say that a neural ODE $\Phi\in\text{NODE}^{k}(\mathcal{X},\mathbb{R})$ has
a bottleneck, if at least one of the weight matrices of the two linear layeres
has not full rank, i.e. $\text{rank}(W)<\min\\{m,n\\}$ or
$\text{rank}(\widetilde{W})=0$. The subset of neural ODEs with a bottleneck is
denoted by

$\textup{NODE}^{k}_{\textup{B}}(\mathcal{X},\mathbb{R})\coloneqq\left\\{\Phi\in\textup{NODE}^{k}(\mathcal{X},\mathbb{R}):\Phi\textup{
has a bottleneck}\right\\}.$

The three different types of neural ODEs are visualized in Figure 4.1. In the
following we show that these three types of architectures build a disjoint
subdivision of all classical feed-forward neural networks.

(a) Structure of a non-augmented neural ODE
$\Phi\in\text{NODE}_{\textup{N}}^{k}(\mathcal{X},\mathbb{R})$, $W$ and
$\widetilde{W}$ have full rank.

(b) Structure of an augmented neural ODE
$\Phi\in\text{NODE}_{\textup{A}}^{k}(\mathcal{X},\mathbb{R})$, $W$ and
$\widetilde{W}$ have full rank.

(c) Structure of a neural network with a bottleneck
$\Phi\in\text{NODE}_{\textup{B}}^{k}(\mathcal{X},\mathbb{R})$, where at least
one of the weight matrices $W$ or $\widetilde{W}$ has not full rank. There is
no relationship between the dimensions $m$ and $n$.

Figure 4.1: The three different types of architectures non-augmented,
augmented and bottleneck for classical feed-forward neural networks
$\Phi\in\text{NODE}^{k}(\mathcal{X},\mathbb{R})$.

######  Proposition 4.4.

The subdivision of neural ODEs in non-augmented neural ODEs of the form
$\textup{NODE}^{k}_{\textup{N}}(\mathcal{X},\mathbb{R})$, augmented neural
ODEs of the form $\textup{NODE}^{k}_{\textup{A}}(\mathcal{X},\mathbb{R})$ and
neural ODEs with a bottleneck
$\textup{NODE}^{k}_{\textup{B}}(\mathcal{X},\mathbb{R})$, is a complete
partition in three disjoint sub-classes of neural ODEs, i.e.

$\textup{NODE}^{k}(\mathcal{X},\mathbb{R})=\textup{NODE}^{k}_{\textup{N}}(\mathcal{X},\mathbb{R})\;\dot{\cup}\;\,\textup{NODE}^{k}_{\textup{A}}(\mathcal{X},\mathbb{R})\;\dot{\cup}\;\,\textup{NODE}^{k}_{\textup{B}}(\mathcal{X},\mathbb{R}).$

###### Proof.

All scalar neural ODEs of the form $\textup{NODE}^{k}(\mathcal{X},\mathbb{R})$
can be split up in neural ODEs, where the weight matrices $W$ and
$\widetilde{W}$ have both full rank or at least one weight matrix has not full
rank and hence the neural ODE is of class
$\textup{NODE}^{k}_{\textup{B}}(\mathcal{X},\mathbb{R})$. The neural ODEs with
full rank matrices are subdivided in the classes
$\textup{NODE}^{k}_{\textup{N}}(\mathcal{X},\mathbb{R})$ and
$\textup{NODE}^{k}_{\textup{A}}(\mathcal{X},\mathbb{R})$, depending on if
$n\geq m$ or if $n<m$, such that we obtain a complete partition in three
disjoint sub-classes of neural ODEs. ∎

######  Remark 4.5.

The comparison of non-augmented and augmented neural ODEs with classical feed-
forward neural networks with full rank matrices is straightforward. Neural
ODEs with a bottleneck resemble neural networks with a bottleneck in the
following sense: if $\textup{rank}(W)=r<\min\\{m,n\\}$, then $L(x)$ has $r$
linearly independent components, such that the effective dimension of the
input is reduced from $n$ to $r$. As the ODE lives in $\mathbb{R}^{m}$, the
dimension is increased from $r$ to $m$, inducing a hidden bottleneck as $n>r$
and $r<m$. In the case that $\textup{rank}(\widetilde{W})=0$, the output of
the neural ODE is constant and has no non-trivial dynamics.

For the upcoming analysis, we define in analogy to classical feed-forward
neural networks a weight space for the weight matrices and biases of scalar
neural ODEs. For neural ODEs, the vector field $f$, which corresponds to all
weights, biases and activation functions of the hidden layer of a feed-forward
neural network, can be freely chosen. As our results do not depend on the
choice of the vector field $f$, we define in the following only a weight space
for the weights and biases $W$, $\widetilde{W}$, $b$ and $\tilde{b}$.

######  Definition 4.6.

Let $\Phi\in\textup{NODE}^{k}(\mathcal{X},\mathbb{R})$ be a scalar neural ODE
with weight matrices $W\in\mathbb{R}^{m\times n}$,
$\widetilde{W}\in\mathbb{R}^{1\times m}$ and biases $b\in\mathbb{R}^{m}$,
$\tilde{b}\in\mathbb{R}$. The weight spaces $\mathbb{V}$ of all possible
weights is defined as

$\mathbb{V}\coloneqq\mathbb{R}^{m\times n}\times\mathbb{R}^{1\times
m}\times\mathbb{R}^{m}\times\mathbb{R}$

The set of stacked weights, where the matrices $W$ and $\widetilde{W}$ have
full rank, is a subset of $\mathbb{R}^{N_{V}}$, $N_{V}\coloneqq mn+2m+1$,
defined by

$\mathbb{V}^{\ast}\coloneqq\left\\{(\textup{vec}(W);\widetilde{W}^{\top};b;\tilde{b})\in\mathbb{R}^{N_{V}}:\textup{rank}(W)=\min\\{m,n\\}\wedge\textup{rank}(\widetilde{W}=1)\right\\},$

with the operator vec of Definition 3.4.

By definition, it holds that the weight matrices of non-augmented and
augmented neural ODEs lie in $\mathbb{V}^{\ast}$ and the weight matrices of
neural ODEs with a bottleneck lie in
$\mathbb{R}^{N_{V}}\setminus\mathbb{V}^{\ast}$. As for feed-forward neural
networks it also holds for neural ODEs that the set of weights
$\mathbb{R}^{N_{V}}\setminus\mathbb{V}^{\ast}$, where at least one weight
matrix has not full rank, is a set of Lebesgue measure zero in
$\mathbb{R}^{N_{V}}$.

######  Lemma 4.7.

The set $\mathbb{R}^{N_{V}}\setminus\mathbb{V}^{\ast}$ is a zero set w.r.t.
the Lebesgue measure in $\mathbb{R}^{N_{V}}$.

###### Proof.

Let $(W,\widetilde{W},b,\tilde{b})\in\mathbb{V}$. By Lemma 3.6, the set

$W_{0}\coloneqq\left\\{\textup{vecm}(W,\widetilde{W})\in\mathbb{R}^{mn+m}:\textup{rank}(W)<\min\\{m,n\\}\vee\textup{rank}(\widetilde{W})=0\right\\}$

is a zero set w.r.t the Lebesgue measure in $\mathbb{R}^{mn+m}$. It follows
that also the set

$W_{0}\times\mathbb{R}^{m}\times\mathbb{R}=\mathbb{R}^{N_{V}}\setminus\mathbb{V}^{\ast}$

is a zero set w.r.t the Lebesgue measure in $\mathbb{R}^{N_{V}}$. ∎

###  4.2 Existence of Critical Points

In this section, we study the existence of critical points dependent on the
special architecture of the scalar neural ODE. As for feed-forward neural
networks, we first calculate the network gradient.

######  Lemma 4.8.

Let $\Phi\in\textup{NODE}^{k}(\mathcal{X},\mathbb{R})$, $k\geq 1$,
$\mathcal{X}\subset\mathbb{R}^{n}$, be a scalar neural ODE with weight
matrices $(W,\widetilde{W},b,\tilde{b})\in\mathbb{V}$. Then

$\nabla\Phi(x)=W^{\top}\left(\frac{\partial h_{s}(T)}{\partial
s}\right)^{\top}\widetilde{W}^{\top}\in\mathbb{R}^{n},$

where $s=Wx+b$ and $\frac{\partial h_{s}(T)}{\partial s}\in\mathbb{R}^{m\times
m}$ is the Jacobian matrix of the time-$T$ map $h_{s}(T)$ w.r.t. the initial
value $s$ of the ODE, i.e. : $\left[\frac{\partial h_{s}(T)}{\partial
s}\right]_{ij}=\frac{\partial[h_{s}(T)]_{i}}{\partial s_{j}}$.

###### Proof.

By Lemma 4.2 it holds $\Phi\in C^{1}(\mathcal{X},\mathbb{R})$, such that the
multi-dimensional chain rule applied to

$\Phi(x)\coloneqq\widetilde{L}(h_{L(x)}(T))=\widetilde{W}\cdot
h_{Wx+b}(T)+\tilde{b}=\widetilde{W}\cdot h_{s}(T)+\tilde{b}$

yields

$\frac{\mathrm{d}\Phi}{\mathrm{d}x}=\widetilde{W}\frac{\partial
h_{s}(T)}{\partial s}W\in\mathbb{R}^{1\times n}$

where $s=Wx+b$. The result follows by taking the transpose. ∎

To determine if a neural ODE has critical points, we need to characterize the
Jacobian matrix $\frac{\partial h_{s}(T)}{\partial s}$. To that purpose, we
need the following two results from ODE theory.

######  Lemma 4.9 ([8]).

Let $f\in C^{k,0}(\mathbb{R}^{m}\times\mathcal{I},\mathbb{R}^{m})$ with $k\geq
1$. Then the solution $h_{s}(t)$ of the initial value problem (IVP) is
continuously differentiable w.r.t. the initial condition $s\in\mathcal{S}$ and
the Jacobian matrix $\frac{\partial h_{s}(t)}{\partial
s}\in\mathbb{R}^{m\times m}$ satisfies the linear variational equation

$\frac{\mathrm{d}Y}{\mathrm{d}t}=\frac{\partial f(h_{s}(t),t)}{\partial
h_{s}}Y,$

with $Y(t)\in\mathbb{R}^{m\times m}$ and initial condition $\frac{\partial
h_{s}(0)}{\partial s}=\textup{Id}_{m}$, where
$\textup{Id}_{m}\in\mathbb{R}^{m\times m}$ is the identity matrix. The matrix
$\frac{\partial f(h_{s}(t),t)}{\partial h_{s}}\in\mathbb{R}^{m\times m}$ is
the Jacobian of the vector field $f(h_{s}(t),t)$ w.r.t. $h_{s}$, defined by
$\left[\frac{\partial f(h_{s}(t),t)}{\partial
h_{s}}\right]_{ij}=\frac{\partial[f(h_{s}(t),t)]_{i}}{\partial[h_{s}]_{j}}$.

######  Lemma 4.10 ([3]).

Consider the linear homogeneous ODE

$\frac{\mathrm{d}y}{\mathrm{d}t}=A(t)y,$ (4.2)

where $y\in\mathbb{R}^{m}$ and $A\in C^{0}(\mathcal{I},\mathbb{R}^{m\times
m})$, $\mathcal{I}\subset\mathbb{R}$ open, is continuous.

  1. (a)

Given an initial condition $y(t_{0})=y_{0}$, $t_{0}\in\mathcal{I}$, then (4.2)
has a unique solution $y:\mathcal{I}\rightarrow\mathbb{R}^{m}$.

  2. (b)

If $Y:\mathcal{I}\rightarrow\mathbb{R}^{m\times m}$ is a matrix solution of
(4.2) with initial condition $Y(t_{0})=Y_{0}\in\mathbb{R}^{m\times m}$, i.e.
every column $[Y]_{i}\in\mathbb{R}^{m}$, $i\in\\{1,\ldots,m\\}$ is a solution
of (4.2) with initial condition $[Y]_{i}(0)=[Y_{0}]_{i}\in\mathbb{R}^{m}$,
then Liouville’s formula holds:

$\det(Y(t))=\det(Y(t_{0}))\exp\left(\int_{t_{0}}^{t}\textup{tr}(A(r))\;\mathrm{d}r\right),$

where $\textup{tr}(A(r))$ is the trace of the matrix $A\in
C^{0}(\mathcal{I},\mathbb{R}^{m\times m})$.

With these two lemmas, we can prove that the Jacobian matrix $\frac{\partial
h_{s}(T)}{\partial s}$ has independently of the choice of the vector field $f$
and the initial condition $s$ always full rank.

######  Proposition 4.11.

Let $\Phi\in\textup{NODE}(\mathcal{X},\mathbb{R})$. Then the Jacobian
$\frac{\partial h_{s}(T)}{\partial s}$ of the time-$T$ map $h_{s}(T)$ w.r.t
the initial data $s\in\mathcal{S}=L(\mathcal{X})$ of the underlying initial
value problem (IVP) has always full rank $m$.

###### Proof.

As $k\geq 1$, the initial value problem (IVP) with initial condition
$s\in\mathcal{S}=L(\mathcal{X})$ has by Lemma 4.1 a unique solution
$h_{s}:\mathcal{I}\rightarrow\mathbb{R}^{m}$ with $[0,T]\subset\mathcal{I}$,
$\mathcal{I}\subset\mathbb{R}$ open. Furthermore, given the continuously
differentiable solution $h_{s}:\mathcal{I}\rightarrow\mathbb{R}^{m}$ of (IVP),
the matrix function $A:\mathcal{I}\rightarrow\mathbb{R}^{m\times m}$,
$A(t)\coloneqq\frac{\partial f(h_{s}(t),t)}{\partial h_{y}}$, is continuous.
By Lemma 4.9, the Jacobian matrix $\frac{\partial h_{s}(t)}{\partial
s}\in\mathbb{R}^{m\times m}$ satisfies the linear homogeneous ODE

$\frac{\mathrm{d}Y}{\mathrm{d}t}=A(t)Y,\qquad Y(0)=\text{Id}_{m},$

where $\text{Id}_{m}\in\mathbb{R}^{m\times m}$ is the identity matrix. By
Lemma 4.10(a), each column of $Y$ defines a unique solution
$[Y]_{i}:\mathcal{I}\rightarrow\mathbb{R}^{m}$ with initial condition
$[Y]_{i}(0)=\mathrm{e}_{i}^{(m)}$, where $\mathrm{e}_{i}^{(m)}$ is the $i$-th
unit vector in $\mathbb{R}^{m}$. Hence,
$Y:\mathcal{I}\rightarrow\mathbb{R}^{m\times m}$ is a unique matrix solution
defined for all $t\in[0,T]\subset\mathcal{I}$ with initial condition
$Y(0)=\text{Id}_{m}$, such that $\det(Y(0))=1$. As the matrix
$A(t)\in\mathbb{R}^{m\times m}$ has only real entries, all eigenvalues of
$A(t)$ are real or they come in complex conjugate pairs, such that the trace
of $A(t)$ is a real number for all $t\in\mathcal{I}$. Consequently,

$\exp\left(\int_{t_{0}}^{t}\textup{tr}(A(r))\;\mathrm{d}r\right)$

is a real-valued, positive number, such that it follows by Lemma 4.10(b) that
$\det(Y(T))\neq 0$. Consequently $Y(T)=\frac{\partial h_{s}(T)}{\partial s}$
has full rank $m$, independently of the initial data $s\in\mathcal{S}$. ∎

The characterization of the Jacobian matrix $\frac{\partial h_{s}(T)}{\partial
s}$ in Proposition 4.11 is sufficient to prove that for non-augmented scalar
ODEs, no critical points can exist.

######  Theorem 4.12.

Any non-augmented scalar neural ODE
$\Phi\in\textup{NODE}_{\textup{N}}(\mathcal{X},\mathbb{R})$, $k\geq 1$ has no
critical point, i.e. $\nabla\Phi(x)\neq 0$ for all $x\in\mathcal{X}$.
Consequently it holds for $k\geq 1$ that

$\textup{NODE}_{\textup{N}}(\mathcal{X},\mathbb{R})\subset(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R}),$

such that every non-augmented scalar neural ODE is a Morse function.

###### Proof.

Denote the weight matrices of $\Phi$ by
$(W,\widetilde{W},b,\tilde{b})\in\mathbb{V}$. By Lemma 4.8 it holds

$\nabla\Phi(x)=W^{\top}\left(\frac{\partial h_{s}(T)}{\partial
s}\right)^{\top}\widetilde{W}^{\top}\in\mathbb{R}^{n}.$

As the neural ODE is non-augmented, $W^{\top}\in\mathbb{R}^{n\times m}$ has
rank $m\leq n$ and $\widetilde{W}^{\top}\in\mathbb{R}^{m\times 1}$ has rank 1.
By Proposition 4.11, the Jacobian $\left(\frac{\partial h_{s}(T)}{\partial
s}\right)^{\top}\in\mathbb{R}^{m\times m}$ has independently of
$s\in\mathcal{S}$ and hence also independently of $x\in\mathcal{X}$ full rank
$m\geq 1$. By Lemma 3.10, the product $\nabla\Phi(x)$ of the three given full
rank matrices with monotonically decreasing widths has also full rank $1$,
which implies that $\nabla\Phi(x)\neq 0$ for all $x\in\mathcal{X}$ and the
result follows. ∎

######  Remark 4.13.

Theorem 4.12 generalizes the results obtained in [15], where it is shown that
a non-augmented neural ODE with $n=m$ and $W=\textup{Id}_{n}$ cannot be of
class $(\mathcal{C}2)^{k}(\mathcal{X},\mathbb{R})$.

The following theorem shows, that the proof of Theorem 4.12 does not apply for
augmented and bottleneck neural ODEs. It guarantees the existence of neural
ODE architectures $\Phi\in\textup{NODE}_{\textup{A}}(\mathcal{X},\mathbb{R})$
and $\Phi\in\textup{NODE}_{\textup{B}}(\mathcal{X},\mathbb{R})$, which do have
critical points.

######  Theorem 4.14.

Let $f\in C^{k,0}(\mathbb{R}^{m}\times\mathcal{I},\mathbb{R}^{m})$, $k\geq 1$
be a vector field, such that $[0,T]\subset\mathcal{I}$ for the initial value
problem (IVP).

  1. (a)

Given $W\in\mathbb{R}^{n\times m}$ with $n<m$ and $\textup{rank}(W)=n$ and a
point $x\in\mathcal{X}$, then the weight matrix
$\widetilde{W}\in\mathbb{R}^{1\times m}$ can be chosen in such a way, that
$\Phi\in\textup{NODE}_{\textup{A}}(\mathcal{X},\mathbb{R})$ has a critical
point at $x$.

  2. (b)

Given $W\in\mathbb{R}^{n\times m}$ with $\textup{rank}(W)<\min\\{m,n\\}$ and a
point $x\in\mathcal{X}$, then the weight matrix
$\widetilde{W}\in\mathbb{R}^{1\times m}$ can be chosen in such a way, that
$\Phi\in\textup{NODE}_{\textup{B}}(\mathcal{X},\mathbb{R})$ has a critical
point at $x$.

###### Proof.

By Lemma 4.8 it holds

$\nabla\Phi(x)=W^{\top}\left(\frac{\partial h_{s}(T)}{\partial
s}\right)^{\top}\widetilde{W}^{\top}\in\mathbb{R}^{n},$

where $s=Wx+b$ for the given $x\in\mathcal{X}$. The Jacobian $\frac{\partial
h_{s}(T)}{\partial s}$ has by Lemma 4.11 full rank $m$.

In case (a), the matrix product $W^{\top}\left(\frac{\partial
h_{s}(T)}{\partial s}\right)^{\top}\in\mathbb{R}^{n\times m}$ has by Lemma
3.10 full rank $n<m$, which implies that the $m$ columns of the matrix product
are linearly dependent. Hence, there exists a non-zero choice of the matrix
$\widetilde{W}\in\mathbb{R}^{1\times m}$, such that $\nabla\Phi(x)=0$ for the
given $x\in\mathcal{X}$.

In case (b), the matrix $W$ has non-full rank, i.e.
$\text{rank}(W)=r<\min\\{m,n\\}$. As the Jacobian matrix $\frac{\partial
h_{s}(T)}{\partial s}$, has full rank, it follows that the matrix product
$W^{\top}\left(\frac{\partial h_{s}(T)}{\partial
s}\right)^{\top}\in\mathbb{R}^{n\times m}$ has also rank $r$. Since
$r<\min\\{m,n\\}$, the $m$ columns of the matrix product are linearly
dependent. Hence, there exists a non-zero choice of the matrix
$\widetilde{W}\in\mathbb{R}^{1\times m}$, such that $\nabla\Phi(x)=0$ for the
given $x\in\mathcal{X}$. ∎

###  4.3 Regularity of Critical Points

For a further characterization of augmented and bottleneck neural ODEs, we
study the regularity of the critical points. A first observation already made
in [15] and [28], is that an augmentation of the phase space in combination
with a linear layer is sufficient, to represent any function exactly. In our
setting this means, that for any map $\Psi\in C^{k}(\mathcal{X},\mathbb{R})$,
$k\geq 0$, there exists an augmented scalar neural ODE
$\Phi\in\text{NODE}^{k}(\mathcal{X},\mathbb{R})$ such that $\Psi(x)=\Phi(x)$
for all $x\in\mathcal{X}$.

######  Theorem 4.15.

Any map $\Psi\in C^{k}(\mathcal{X},\mathbb{R})$, $k\geq 0$,
$\mathcal{X}\subset\mathbb{R}^{n}$ can be exactly represented by an augmented
scalar neural ODE
$\Phi\in\textup{NODE}^{k}_{\textup{A}}(\mathcal{X},\mathbb{R})$ with arbitrary
$m>n$, i.e. $\Psi(x)=\Phi(x)$ for all $x\in\mathcal{X}$.

###### Proof.

In analogy to the proof in [14], we fix $T>0$, choose $m>n$ and define the
augmented neural ODE $\Phi$ by the weights

$W=\begin{pmatrix}\textup{Id}_{n}\\\ 0\end{pmatrix}\in\mathbb{R}^{m\times
n},\quad
b=0\in\mathbb{R}^{m},\quad\widetilde{W}=(0,\ldots,0,1)\in\mathbb{R}^{1\times
m},\quad\tilde{b}=0\in\mathbb{R}$

and the ODE

$\frac{\mathrm{d}}{\mathrm{d}t}\begin{pmatrix}h_{1}\\\ \vdots\\\ h_{m-1}\\\
h_{m}\end{pmatrix}=\begin{pmatrix}0\\\ \vdots\\\ 0\\\
\frac{1}{T}\cdot\Psi(h_{1},\ldots.h_{n})\end{pmatrix}\eqqcolon
f(h_{1},\ldots,h_{n},t),$

such that the vector field $f$ is of class
$C^{k,0}(\mathcal{X}\times\mathbb{R},\mathbb{R})$. $\Phi$ is augmented, as
$m>n$ and the weights matrices $W$ and $\widetilde{W}$ have full rank. By
Lemma 4.2 it holds
$\Phi\in\textup{NODE}^{k}_{\textup{A}}(\mathcal{X},\mathbb{R})$ and it follows

$\Phi(x)=\widetilde{W}\cdot h_{Wx+b}(T)+\tilde{b}=\widetilde{W}\cdot
h_{(x,0)^{\top}}(T)=\widetilde{W}\begin{pmatrix}x\\\ 0\\\
\Psi(x)\end{pmatrix}=\Psi(x)$

for all $x\in\mathcal{X}$. ∎

The last theorem implies, that any function $\Psi\in
C^{k}(\mathcal{X},\mathbb{R})$, $k\geq 0$ can be represented exactly by an
augmented neural ODE. If $k\geq n+1\geq 2$, Morse functions are by Theorem 2.2
a dense subset in the Banach space of $k$ times differentiable functions.
Theorem 4.15 does not characterize, if Morse functions are also dense in the
space of augmented neural ODEs, i.e. if it is a generic property of a map
$\Phi\in\textup{NODE}^{k}_{\textup{A}}(\mathcal{X},\mathbb{R})$ to be a Morse
function. In analogy to classical feed-forward neural networks, we show in the
following that this is the case for all weights, except possibly for a zero
set with respect to the Lebesgue measure.

######  Theorem 4.16.

Any augmented scalar neural ODE
$\Phi\in\textup{NODE}^{k}_{\textup{A}}(\mathcal{X},\mathbb{R})$, $k\geq 2$ is
for all weights
$(\textup{vec}(W);\widetilde{W}^{\top};b;\tilde{b})\in\mathbb{V}^{\ast}$,
except possibly for a zero set in $\mathbb{R}^{N_{V}}$ w.r.t. the Lebesgue
measure, of class $(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$ or
$(\mathcal{C}2)^{k}(\mathcal{X},\mathbb{R})$ and hence a Morse function.

###### Proof.

Let $\Phi\in\textup{NODE}^{k}_{\textup{A}}(\mathcal{X},\mathbb{R})$ be an
augmented scalar neural ODE with full rank weight matrices
$W\in\mathbb{R}^{m\times n}$, $\widetilde{W}\in\mathbb{R}^{1\times m}$ and
biases $b\in\mathbb{R}^{n}$, $\tilde{b}\in\mathbb{R}$. By Theorem 4.14(a) it
is possible that the neural ODE $\Phi$ has a critical point. In the case that
$\Phi$ has no critical point, $\Phi$ is of class
$(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$ and hence a Morse function. In
the following we show, that if $\Phi$ has a critical point, then it is for all
weights, where $\Phi$ has critical points, except possibly for a zero set
w.r.t. the Lebesgue measure in $\mathbb{R}^{N_{V}}$ of class
$(\mathcal{C}2)^{k}(\mathcal{X},\mathbb{R})$. By Lemma 4.8, the gradient of
$\Phi$ is given by

$\nabla\Phi(x)=W^{\top}\left(\frac{\partial h_{s}(T)}{\partial
s}\right)^{\top}\widetilde{W}^{\top}\in\mathbb{R}^{n},$

where $s=Wx+b$ and $\frac{\partial h_{s}(T)}{\partial s}\in\mathbb{R}^{m\times
m}$ is the Jacobian matrix of the time-$T$ map $h_{s}(T)$ w.r.t. the initial
value $s$. For a weight vector $v\in\mathbb{V}^{\ast}$, we define
$\widehat{\Phi}\in C^{k}(\mathcal{X}\times\mathbb{V}^{\ast})$, $k\geq 2$, to
be the neural ODE $\Phi$ with an explicit dependence on the weight vector $v$.
As a composition of $k$ times continuously differentiable functions,
$\widehat{\Phi}$ is not only in $x$ but also in $v$ $k$ times continuously
differentiable. We aim to apply Lemma 3.23 to show that $\Phi$ is for all
weights $v\in\mathbb{V}^{\ast}$, except possibly for a set of measure zero
w.r.t. the Lebesgue measure in $\mathbb{R}^{N_{V}}$ a Morse function. To that
purpose, we need to show that the matrix
$\frac{\partial^{2}\widehat{\Phi}}{\partial v\partial x}\in\mathbb{R}^{n\times
p}$ has for every $(x,v)\in\mathcal{X}\times\mathbb{V}^{\ast}$ full rank. The
second partial derivatives with respect to $x$ and the components of
$\widetilde{W}$ are given by

$\frac{\partial^{2}\widehat{\Phi}}{\partial[\widetilde{W}]_{i}\partial
x}=\frac{\partial}{\partial[\widetilde{W}]_{i}}\left[W^{\top}\left(\frac{\partial
h_{s}(T)}{\partial
s}\right)^{\top}\widetilde{W}^{\top}\right]=W^{\top}\left(\frac{\partial
h_{s}(T)}{\partial s}\right)^{\top}\mathrm{e}_{i}^{(m)},$

where $\mathrm{e}_{i}^{(m)}$ denotes the $i$-th unit vector in
$\mathbb{R}^{m}$. Consequently it follows that

$\frac{\partial^{2}\widehat{\Phi}}{\partial\widetilde{W}\partial
x}=W^{\top}\left(\frac{\partial h_{s}(T)}{\partial
s}\right)^{\top}\in\mathbb{R}^{n\times m},$

has full rank $n$, since by assumption the matrix $W$ has full rank $n$ as
$m>n$ and by Proposition 4.11 the Jacobian $\frac{\partial h_{s}(T)}{\partial
s}$ has full rank $m$. Hence, the weight matrix
$\frac{\partial^{2}\widehat{\Phi}}{\partial v\partial x}\in\mathbb{R}^{n\times
p}$ has for every $(x,v)\in\mathcal{X}\times\mathbb{V}^{\ast}$ full rank
$n<p$, because the submatrix
$\frac{\partial^{2}\widehat{\Phi}}{\partial\widetilde{W}\partial x}$ has rank
$n$. Lemma 3.23 implies, that $\Phi$ is for all weights
$v\in\mathbb{V}^{\ast}$, except possibly for a zero set w.r.t. the Lebesgue
measure in $\mathbb{R}^{N_{V}}$ a Morse function. ∎

We illustrate the assertion of the last theorem in a one-dimensional example.

######  Example 4.17.

Consider the scalar augmented neural ODE
$\Phi\in\textup{NODE}^{\infty}_{\textup{A}}(\mathbb{R},\mathbb{R})$ with
$n=1$, $m=2$ defined by the weights and biases

$W=\begin{pmatrix}w_{1}\\\ w_{2}\end{pmatrix}\in\mathbb{R}^{2},\qquad
b=\begin{pmatrix}b_{1}\\\
b_{2}\end{pmatrix}\in\mathbb{R}^{2},\qquad\widetilde{W}=(\widetilde{w}_{1},\widetilde{w}_{2})\in\mathbb{R}^{1\times
2},\qquad\tilde{b}\in\mathbb{R},$

which is visualized in Figure 4.2. As underlying initial value problem we
consider

$\frac{\mathrm{d}}{\mathrm{d}t}\begin{pmatrix}h_{1}\\\
h_{2}\end{pmatrix}=\begin{pmatrix}0\\\
\exp(h_{1})\end{pmatrix},\qquad\begin{pmatrix}h_{1}(0)\\\
h_{2}(0)\end{pmatrix}=Wx+b,$

with solution

$\begin{pmatrix}h_{1}(t)\\\
h_{2}(t)\end{pmatrix}=\begin{pmatrix}w_{1}x+b_{1}\\\
w_{2}x+b_{2}+\exp(w_{1}x+b_{1})t\end{pmatrix},$

such that it follows

$\displaystyle\Phi(x)$
$\displaystyle=\tilde{w}_{1}(w_{1}x+b_{1})+\tilde{w}_{2}(w_{2}x+b_{2}+\exp(w_{1}x+b_{1})T),$
$\displaystyle\nabla\Phi(x)$
$\displaystyle=\tilde{w}_{1}w_{1}+\tilde{w}_{2}w_{2}+\tilde{w}_{2}w_{1}\exp(w_{1}x+b_{1})T,$
$\displaystyle H_{\Phi}(x)$
$\displaystyle=\tilde{w}_{2}w_{1}^{2}\exp(w_{1}x+b_{1})T.$

Consequently, $\Phi$ can only be of class
$(C3)^{\infty}(\mathbb{R},\mathbb{R})$ if

$(\textup{vec}(W);\widetilde{W}^{\top};b;\tilde{b})\in\left\\{(w_{1},w_{2},\tilde{w}_{1},\tilde{w}_{2},b_{1},b_{2},\tilde{b})\in\mathbb{V}^{\ast}\subset\mathbb{R}^{7}:\tilde{w}_{2}=0\vee
w_{1}=0\right\\},$

which is a zero set w.r.t. the Lebesgue measure in $\mathbb{R}^{7}$. Hence for
all weights
$(\textup{vec}(W);\widetilde{W}^{\top};b;\tilde{b})\in\mathbb{V}^{\ast}$,
except possibly for a zero set in $\mathbb{R}^{7}$ w.r.t. the Lebesgue
measure, $\Phi$ is of class $(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$ or
$(\mathcal{C}2)^{k}(\mathcal{X},\mathbb{R})$.

Figure 4.2: Scalar augmented neural ODE
$\Phi\in\textup{NODE}^{\infty}_{\textup{A}}(\mathbb{R},\mathbb{R})$ of Example
4.17.

For neural ODEs with a bottleneck, we cannot guarantee that the considered
matrix $\frac{\partial^{2}\widehat{\Phi}}{\partial v\partial x}$ in the proof
of Theorem 4.16 has always full rank. In the following we can directly show,
that any critical point of a neural ODE with a bottleneck has to be
degenerate. To that purpose, we derive a formula for the Hessian matrix
$H_{\Phi}(x)$.

######  Lemma 4.18.

Let $\Phi\in\textup{NODE}^{k}(\mathcal{X},\mathbb{R})$, $k\geq 2$,
$\mathcal{X}\subset\mathbb{R}^{n}$ be a scalar neural ODE with weight matrices
$(W,\widetilde{W},b,\tilde{b})\in\mathbb{V}$. Then

$H_{\Phi}(x)=W^{\top}H_{\eta}(s)W\in\mathbb{R}^{n\times n},$

where $s=L(x)=Wx+b$ and $\eta\in C^{k}(\mathcal{S},\mathbb{R})$,
$\eta(s)\coloneqq\widetilde{W}h_{s}(T)$, $\mathcal{S}=L(\mathcal{X})$.

###### Proof.

We introduce $\eta:\mathcal{S}\rightarrow\mathbb{R}$,
$\eta(s)\coloneqq\widetilde{W}h_{s}(T)$, which is by Lemma 4.1 of class
$C^{k}(\mathcal{S},\mathbb{R})$ with $\mathcal{S}=L(\mathcal{X})$. It follows

$\nabla\eta(x)=\frac{\partial h_{s}(T)}{\partial
s}^{\top}\widetilde{W}^{\top}\in\mathbb{R}^{m},$

such that Lemma 4.8 implies $\nabla\Phi(x)=W^{\top}\nabla\eta(x)$. Hence, the
multi-dimensional chain rule implies

$\displaystyle
H_{\Phi}(x)=\nabla(\nabla\Phi(x)^{\top})=\nabla(\nabla\eta(x)^{\top}W)=W^{\top}H_{\eta}(s)W\in\mathbb{R}^{n\times
n},$

where $s=Wx+b$. ∎

Without any analysis of the Hessian matrix $H_{\eta}$, we can directly
conclude, that every critical point of a neural ODE with a bottleneck is
degenerate, as at least one of the weight matrices $W$ or $\widetilde{W}$ is
singular.

######  Theorem 4.19.

Any scalar neural ODE with a bottleneck
$\Phi\in\textup{NODE}^{k}_{\textup{B}}(\mathcal{X},\mathbb{R})$ has only
degenerate critical points, i.e. $\Phi$ is of class
$(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$ or
$(\mathcal{C}3)^{k}(\mathcal{X},\mathbb{R})$.

###### Proof.

As the scalar neural ODE $\Phi$ has a bottleneck, at least one of the weight
matrices $W\in\mathbb{R}^{m\times n}$ and $\widetilde{W}\in\mathbb{R}^{1\times
m}$ has not full rank. If $\widetilde{W}=0$, then $\Phi(x)=\tilde{b}$ is
constant, such that every point $x\in\mathcal{X}$ is a degenerate critical
point and hence $\Phi\in(\mathcal{C}3)^{k}(\mathcal{X},\mathbb{R})$. If
$\textup{rank}(W)<\min\\{m,n\\}$, then it holds

$\textup{rank}(H_{\Phi}(x))\leq\min\\{\textup{rank}(W^{\top}),\textup{rank}(H_{\eta}(s)),\textup{rank}(W)\\}<\min\\{m,n\\},$

which implies that the Hessian matrix $H_{\Phi}(x)$ is singular. Consequently
if $\Phi$ has a critical point, then
$\Phi\in(\mathcal{C}3)^{k}(\mathcal{X},\mathbb{R})$ and if $\Phi$ has no
critical point, then $\Phi\in(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$. ∎

In Theorem 3.28 we showed, that feed-forward neural networks with a bottleneck
can be of all classes $(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$,
$(\mathcal{C}2)^{k}(\mathcal{X},\mathbb{R})$ and
$(\mathcal{C}3)^{k}(\mathcal{X},\mathbb{R})$. The following remark explains,
why nevertheless, the results established for neural networks and neural ODEs
are comparable.

######  Remark 4.20.

It is for both scalar neural ODEs and feed-forward neural networks the case
that if the last layer $\widetilde{W}\in\mathbb{R}^{1\times m}$ or
$\widetilde{W}_{L}\in\mathbb{R}^{1\times m_{L}}$ has not full rank, the output
is constant and hence the network is of class
$(C3)^{k}(\mathcal{X},\mathbb{R})$. We can compare the case that the weight
matrix $W\in\mathbb{R}^{m\times n}$ of the neural ODE has not full rank with
the case, that the first weight matrix $W_{1}\in\mathbb{R}^{m_{1}\times n}$ of
the feed-forward neural network has not full rank. If $W_{1}$ is singular,
still all four types of bottlenecks in Theorem 3.28 are possible, but Theorem
3.15 implies, that the neural network cannot be of class
$(\mathcal{C}2)^{k}(\mathcal{X},\mathbb{R})$. This is the case, as a linear
change of coordinates to obtain an equivalent neural network architecture is
by the proof of Theorem 3.12 needed if and only if the first weight matrix
$W_{1}$ has not full rank. The difference of the results of neural ODEs and
feed-forward neural networks is induced by the fact, that for neural ODEs the
bottleneck obstruction cannot occur in the initial value problem, which
corresponds to the hidden layer of the neural network.

For the more interesting case, that the first weight matrix
$W\in\mathbb{R}^{m\times n}$ of the scalar neural ODE has not full rank, we
provide an example which illustrates that the neural ODE can be both of class
$(\mathcal{C}1)^{k}(\mathcal{X},\mathbb{R})$ or
$(\mathcal{C}3)^{k}(\mathcal{X},\mathbb{R})$.

######  Example 4.21.

Consider the scalar neural ODE with a bottleneck
$\Phi\in\textup{NODE}^{k}_{\textup{B}}(\mathbb{R},\mathbb{R})$, $k\geq 2$ with
$n=2$, $m=2$ defined by the weights and biases

$W=\begin{pmatrix}w_{11}&w_{12}\\\
w_{21}&w_{22}\end{pmatrix}\in\mathbb{R}^{2\times 2},\qquad
b=\begin{pmatrix}b_{1}\\\
b_{2}\end{pmatrix}\in\mathbb{R}^{2},\qquad\widetilde{W}=(\widetilde{w}_{1},\widetilde{w}_{2})\in\mathbb{R}^{1\times
2},\qquad\tilde{b}\in\mathbb{R},$

where we assume that the bottleneck is induced by the condition
$\textup{rank}(W)=1$. $\Phi$ is visualized in Figure 4.3. As
$\textup{rank}(W)=1$, there exists a constant $\lambda$, such that

$\begin{pmatrix}w_{21}&w_{22}\end{pmatrix}=\lambda\cdot\begin{pmatrix}w_{11}&w_{12}\end{pmatrix}\qquad\Rightarrow\qquad
s=\begin{pmatrix}s_{1}\\\ s_{2}\end{pmatrix}=Wx+b=\begin{pmatrix}q+b_{1}\\\
\lambda q+b_{2}\end{pmatrix}$

with $q\coloneqq w_{11}x_{1}+w_{12}x_{2}$. Consequently it holds

$\Phi(x)=\tilde{w}_{1}[h_{s}(T)]_{1}+\tilde{w}_{2}[h_{s}(T)]_{2}$

and it follows for the gradient

$\nabla\Phi(x)=\begin{pmatrix}w_{11}\left(\frac{\partial\Phi}{\partial
s_{1}}+\lambda\frac{\partial\Phi}{\partial s_{2}}\right)\\\
w_{12}\left(\frac{\partial\Phi}{\partial
s_{1}}+\lambda\frac{\partial\Phi}{\partial s_{2}}\right)\end{pmatrix}.$

As the columns of $\nabla\Phi$ are linearly dependent, also the columns of the
Hessian $H_{\Phi}$ are linearly dependent, which implies that every possible
critical point of $\Phi$ is degenerate. Dependent on the vector field and on
$\lambda$, $\Phi$ has or does not have critical points. As an example we
consider the underlying initial value problem defined by

$\frac{\mathrm{d}}{\mathrm{d}t}\begin{pmatrix}h_{1}\\\
h_{2}\end{pmatrix}=\begin{pmatrix}h_{1}\\\
h_{2}\end{pmatrix},\qquad\begin{pmatrix}h_{1}(0)\\\
h_{2}(0)\end{pmatrix}=\begin{pmatrix}s_{1}\\\
s_{2}\end{pmatrix}\qquad\Rightarrow\qquad\begin{pmatrix}h_{1}(T)\\\
h_{2}(T)\end{pmatrix}=\begin{pmatrix}s_{1}e^{T}\\\ s_{2}e^{T}\end{pmatrix},$

such that

$\nabla\Phi(x)=\begin{pmatrix}w_{11}\left(\tilde{w}_{1}+\lambda\tilde{w}_{2}\right)e^{T}\\\
w_{12}\left(\tilde{w}_{1}+\lambda\tilde{w}_{2}\right)e^{T}\end{pmatrix}.$

Hence, for $\lambda=-\frac{\tilde{w}_{1}}{\tilde{w}_{2}}$ or for $W=0$, every
point $x\in\mathcal{X}$ is a degenerate critical point and $\Phi$ is of class
$(\mathcal{C}3)^{\infty}(\mathcal{X},\mathbb{R})$, otherwise $\Phi$ has no
critical point and is of class
$(\mathcal{C}1)^{\infty}(\mathcal{X},\mathbb{R})$.

Figure 4.3: Scalar neural ODE with a bottleneck
$\Phi\in\textup{NODE}^{k}_{\textup{B}}(\mathbb{R},\mathbb{R})$ of Example
4.21.

##  5 Conclusion and Outlook

In this work, we studied the geometric structure of the input-output map of
feed-forward neural networks and neural ODEs. We used the definition of a
Morse function, which is a scalar function, where every critical point is non-
degenerate. It is interesting to study the property of being a Morse function,
as Morse functions are dense in the space of sufficiently smooth scalar
functions and are fundamental building blocks within many areas of
mathematics. We fully classified the existence and regularity of critical
points depending on the specific architecture of the feed-forward neural
network or the neural ODE.

The architectures studied are subdivided in non-augmented, augmented and
bottleneck architectures. For classical feed-forward neural networks, the
subdivision is defined by width of the largest layer and by checking if there
exists a bottleneck in the underlying graph. If at least one weight matrix of
a neural network $\Phi$ has not full rank, there exists up to a linear change
of coordinates a normal form $\overline{\Phi}$, which is a feed-forward neural
network with only full rank matrices and which is equivalent to $\Phi$, i.e.
it has the same input-output map. Our proof of the existence of a normal form
is constructive, i.e. we gave an explicit algorithm to calculate an equivalent
normal form $\overline{\Phi}$. Furthermore we proved that the case that all
weight matrices have full rank is generic, i.e. the set of weights, where at
least one weight matrix has not full rank has Lebesgue measure zero in the
weight space. Hence, we focused in the following analysis on neural networks
in normal form.

For neural ODEs, we defined an analogous subdivision in non-augmented,
augmented and bottleneck architectures. As neural ODEs can be interpreted as
an infinite depth limit of classical feed-forward neural networks, the vector
field $f$ of the neural ODE represents all weights, biases and activation
functions of the hidden layers. We studied the most general case, where $f$
can be any non-autonomous vector field. In analogy to feed-forward neural
networks, the dimension of the phase space defines if an architecture is
augmented or non-augmented. We say that a neural ODE has a bottleneck, if at
least one of the weight matrices of the linear layers before and after the
initial value problem has not full rank.

Our main contribution of this work is to classify in each case the existence
and regularity of critical points. In both non-augmented cases, no critical
points can exist, as the network gradient has always full rank, which we
proved via rank arguments and linear variational equations. If the dimension
of at least one layer or the dimension of the phase space is augmented, we
showed that it is in both cases possible that critical points exist. Using
differential geometry and Morse theory we showed, that except for a zero set
with respect to the Lebesgue measure in the weight space, the input-output map
is a Morse function. For feed-forward neural networks with a bottleneck, we
showed that critical points can be degenerate or non-degenerate. Depending on
the dimension of the first bottleneck and the weight matrices, we derived
conditions, specifying the regularity of the critical points depending on the
type of bottleneck. In contrast to that, we proved for neural ODEs with a
bottleneck, that every critical points is degenerate. The different
classification for bottleneck architectures relies on the fact, that for
neural ODEs the bottleneck cannot occur in the hidden layer. We explain, why
nevertheless the results obtained for neural ODEs with a bottleneck are
comparable to the results for feed-forward neural networks, where the first
weight matrix has not full rank. Overall our results show, that the geometric
structure of the input-output map is comparable for feed-forward neural
networks and for neural ODEs.

Besides the classification of critical points, we also studied the
implications of our results on the universal embedding property of feed-
forward neural networks and neural ODEs. For neural ODEs we showed that they
cannot have the universal embedding property in the non-augmented and in the
bottleneck case, whereas augmented neural ODEs have the universal embedding
property. Hereby it was used, that $f$ can be any non-autonomous vector field.
In the non-augmented case, feed-forward neural networks cannot have the
universal embedding property. For augmented feed-forward neural networks,
there exist already various universal approximation theorems in the
literature. For the bottleneck case, we distinguished between cases, where no
universal embedding is possible, and cases, where the input-output dynamics
resembles augmented neural networks, such that it is possible that the neural
network has the universal approximation property.

The focus on the exact output of feed-forward neural networks and neural ODEs
explains, why certain architectures perform better than others. Our analysis
showed, how the dimensions of the layers and the phase space, the rank of the
weight matrices, and obstructions such as bottlenecks influence if the input-
output map is a Morse function. The established results could be used as a
starting point for a perturbation analysis to derive universal approximation
results. Furthermore, in future work it would be interesting to study the
geometric implications if a feed-forward neural network or a neural ODE is a
Morse function. Morse functions are used across many areas, such as topology,
differential geometry and data analysis, to study the underlying manifold, on
which the Morse function is defined. Transferred to our setting, it is of
interest to study the manifold of input data or initial conditions with an
augmented neural network defined on it, which is generically a Morse function.
Beyond the evident connection to data geometry and its applications, one could
even aim to consider from a very theoretical viewpoint, whether neural network
representations of Morse functions could be helpful to solve open conjectures
in Morse, Morse-Bott and Floer homology.

Acknowledgments: CK and SVK would like to thank the DFG for partial support
via the SPP2298 ‘Theoretical Foundations of Deep Learning’. CK would like to
thank the VolkswagenStiftung for support via a Lichtenberg Professorship. SVK
would like to thank the Munich Data Science Institute (MDSI) for partial
support via a Linde doctoral fellowship.

## References

  * [1] C. C. Aggarwal.  Neural Networks and Deep Learning.  Springer, 1 edition, 2018.  doi:10.1007/978-3-319-94463-0.
  * [2] R. T. Q. Chen, Y. Rubanova, J. Bettencourt, and D. Duvenaud.  Neural ordinary differential equations.  2018\.  doi:10.48550/ARXIV.1806.07366.
  * [3] C. Chicone.  Ordinary Differential Equations with Applications, volume 34 of Texts in Applied Mathematics.  Springer New York, 2 edition, 2006.  doi:10.1007/0-387-35794-7.
  * [4] B. J. Cook, A. D. H. Peterson, W. Woldman, and J. R. Terry.  Neural field models: A mathematical overview and unifying framework.  Mathematical Neuroscience and Applications, Volume 2, 2022.  doi:10.46298/mna.7284.
  * [5] C. Esteve, B. Geshkovski, D. Pighin, and E. Zuazua.  Large-time asymptotics in deep learning.  2020\.  doi:10.48550/ARXIV.2008.02491.
  * [6] O. Forster.  Analysis 2, Differentialrechnung im $\mathbb{R}^{n}$, gewöhnliche Differentialgleichungen.  Grundkurs Mathematik. Springer Spektrum, 11 edition, 2017.  doi:10.1007/978-3-658-19411-6.
  * [7] J. Guckenheimer and P. Holmes.  Nonlinear Oscillations, Dynamical Systems, and Bifurcations of Vector Fields, volume 42 of Applied Mathematical Sciences.  Springer New York, 7 edition, 2002.  doi:10.1007/978-1-4612-1140-2.
  * [8] J. K. Hale.  Ordinary Differential Equations.  Krieger Publishing Company, 2 edition, 1980.
  * [9] K. He, X. Zhang, S. Ren, and J. Sun.  Deep residual learning for image recognition.  IEEE Conference on Computer Vision and Pattern Recognition, pages 770–778, jun 2016.  doi:10.1109/cvpr.2016.90.
  * [10] M. W. Hirsch.  Differential Topology, volume 33 of Graduate Texts in Mathematics.  Springer New York, 1976.  doi:10.1007/978-1-4684-9449-5.
  * [11] K. Hornik, M. Stinchcombe, and H. White.  Multilayer feedforward networks are universal approximators.  Neural Networks, 2(5):359–366, jan 1989.  doi:10.1016/0893-6080(89)90020-8.
  * [12] P. Kidger.  On Neural Differential Equations.  PhD thesis, Mathematical Institute, University of Oxford, 2022.  doi:10.48550/ARXIV.2202.02435.
  * [13] A. Kratsios.  The universal approximation property - characterization, construction, representation, and existence.  Annals of Mathematics and Artificial Intelligence, 89(5-6):435–469, jan 2021.  doi:10.1007/s10472-020-09723-1.
  * [14] C. Kuehn.  PDE Dynamics: An Introduction.  Society for Industrial and Applied Mathematics, 2019.  doi:10.1137/1.9781611975666.
  * [15] C. Kuehn and S.-V. Kuntz.  Embedding capabilities of neural ODEs.  2023\.  doi:10.48550/ARXIV.2308.01213.
  * [16] S. V. Kurochkin.  Neural network with smooth activation functions and without bottlenecks is almost surely a morse function.  Computational Mathematics and Mathematical Physics, 61(7):1162–1168, 2021.  doi:10.1134/s0965542521070101.
  * [17] H. Lin and S. Jegelka.  ResNet with one-neuron hidden layers is a universal approximator.  Advances in Neural Information Processing Systems, 31:6169–6178, 2018.  doi:10.48550/ARXIV.1806.10909.
  * [18] J. R. Magnus and H. Neudecker.  Matrix Differential Calculus with Applications in Statistics and Econometrics.  Wiley, 3 edition, 2019.  doi:10.1002/9781119541219.
  * [19] M. Morse.  The Calculus of Variations in the Large, volume 18 of Colloquium Publications.  American Mathematical Society, 1934.
  * [20] L. Nicolaescu.  An Invitation to Morse Theory.  Springer New York, 2011.  doi:10.1007/978-1-4614-1105-5.
  * [21] A. Pinkus.  Approximation theory of the MLP model in neural networks.  Acta Numerica, 8:143–195, jan 1999.  doi:10.1017/s0962492900002919.
  * [22] V. Prasolov.  Elements of Combinatorial and Differential Topology.  American Mathematical Society, 2006.  doi:10.1090/gsm/074.
  * [23] F. Rosenblatt.  The perceptron - a perceiving and recognizing automaton.  Cornell Aeronautical Laboratory, INC., Buffalo, New York, (85-460-1), 1957.
  * [24] A. M. Schäfer and H. G. Zimmermann.  Recurrent Neural Networks Are Universal Approximators, pages 632–640.  Springer Berlin Heidelberg, 2006.  doi:10.1007/11840817_66.
  * [25] I. R. Shafarevich and A. O. Remizov.  Linear Algebra and Geometry.  Springer Berlin Heidelberg, 2013.  doi:10.1007/978-3-642-30994-6.
  * [26] N. Thome.  Inequalities and equalities for l=2 (sylvester), l=3 (frobenius), and l¿3 matrices.  Aequationes mathematicae, 90(5):951–960, 2016.  doi:10.1007/s00010-016-0412-4.
  * [27] E. Weinan.  A proposal on machine learning via dynamical systems.  Commun. Math. Stat, 5:1–11, mar 2017.  doi:10.1007/s40304-017-0103-z.
  * [28] H. Zhang, X. Gao, J. Unterman, and T. Arodz.  Approximation capabilities of neural odes and invertible residual networks.  Proceedings of the 37th International Conference on Machine Learning, 119:11086–11095, 2020.  doi:10.48550/ARXIV.1907.12998.