{\mu}, g)$.

Let $\boldsymbol{\mu}_0\in\Delta(\mathcal{X})$ be the distribution of initial local states and $g_0$ be the initial global state. Define $V_{\infty}^*(\boldsymbol{\mu}_0, g_0)\triangleq \sup_{\Phi\in\mathbb{R}^{\mathrm{d}}} V_{\infty}(\boldsymbol{\mu}_0, g_0, \pi_{\Phi})$ to be the optimal mean-field value obtained over the set of feasible policies, $\Pi$ corresponding to the initial condition $(\boldsymbol{\mu}_0, g_0)$.  Assume $\{\Phi_j\}_{j=1}^J$ to be a sequence of $\mathrm{d}-$dimensional parameters generated by the NPG algorithm \citep{liu2020improved, agarwal2021theory} as follows.
\begin{equation}
\label{npg_update}
\Phi_{j+1} = \Phi_j + \eta \mathbf{w}_j,  \mathbf{w}_j\triangleq{\arg\min}_{\mathbf{w}\in\mathbb{R}^{\mathrm{d}}} ~L_{ \zeta_{(\boldsymbol{\mu}_0,g_0)}^{\Phi_j}}(\mathbf{w},\Phi_j)
\end{equation}
where $\eta>0$ is the learning parameter. The function $L_{\zeta_{(\boldsymbol{\mu}_0,g_0)}^{\Phi_j}}$ and the occupancy measure $\zeta_{(\boldsymbol{\mu}_0, g_0)}^{\Phi_j}$ are defined below. 
\begin{align}
\begin{split}
L_{ \zeta_{(\boldsymbol{\mu}_0, g_0)}^{\Phi'}}(\mathbf{w},\Phi)\triangleq &\mathbb{E}_{(x,\boldsymbol{\mu},g,u)\sim \zeta_{(\boldsymbol{\mu}_0, g_0)}^{\Phi'}}\Big[\Big(A_{\Phi}(x,\boldsymbol{\mu},g,u)
-(1-\gamma)\mathbf{w}^{\mathrm{T}}\nabla_{\Phi}\log \pi_{\Phi}(x,\boldsymbol{\mu}, g)(u) \Big)^2\Big],
\end{split}\\
\label{zeta_dist}
\begin{split}
\zeta_{(\boldsymbol{\mu}_0, g_0)}^{\Phi'}(x,\boldsymbol{\mu},g, u)
&\triangleq \sum_{\tau=0}^{\infty}\gamma^{\tau} \mathbb{P}(x_\tau=x,\boldsymbol{\mu}_{\tau}=\boldsymbol{\mu}, g_{\tau}=g, u_\tau=u
|x_0=x,\boldsymbol{\mu}_0=\boldsymbol{\mu},g_0=g,u_0=u,\boldsymbol{\pi}_{\Phi'})(1-\gamma)
\end{split}
\end{align}


Note that in order to calculate the update direction, $\mathbf{w}_j$ at the $j$-th NPG update $(\ref{npg_update})$, we must solve another optimization problem over $\mathbb{R}^{\mathrm{d}}$. The second minimization problem can be handled via a stochastic gradient descent (SGD) approach. Particularly, for a given $\Phi_j\in\mathbb{R}^{\mathrm{d}}$, the minimizer of $L_{ \zeta_{(\boldsymbol{\mu}_0, g_0)}^{\Phi_j}}(\cdot,\Phi)$ can be obtained via the following SGD updates: $\mathbf{w}_{j,l+1}=\mathbf{w}_{j,l}-\alpha\mathbf{h}_{j,l}$ \citep{liu2020improved} where $\alpha>0$ is the learning parameter for the sub-problem and $\mathbf{h}_{j,l}$, the gradient at the $l$-th iteration is given as follows.
\begin{align}
\begin{split}
\mathbf{h}_{j,l}&\triangleq \Bigg(\mathbf{w}_{j,l}^{\mathrm{T}}\nabla_{\Phi_j}\log \pi_{\Phi_j}(x,\boldsymbol{\mu}, g)(u)-\dfrac{1}{1-\gamma}\hat{A}_{\Phi_j}(x,\boldsymbol{\mu},g,u)\Bigg)
\nabla_{\Phi_j}\log \pi_{\Phi_j}(x,\boldsymbol{\mu}, g)(u)
\end{split}
\label{sub_prob_grad_update}
\end{align}
where the tuple $(x,\boldsymbol{\mu},g,u)$ is sampled from the occupancy measure $\zeta_{\boldsymbol{\lambda}_0}^{\Phi_j}$, and $\hat{A}_{\Phi_j}(x, \boldsymbol{\mu}, g, u)$ denotes a unbiased estimator of $A_{\Phi_j}(x, \boldsymbol{\mu}, g, u)$. The sampling procedure and the method to obtain the estimator has been described in Algorithm \ref{algo_2} in Appendix \ref{sampling_process}. It is to be clarified that Algorithm \ref{algo_2} is based on Algorithm 3 of \citep{agarwal2021theory}. The whole NPG procedure is summarized in Algorithm \ref{algo_1}.

\begin{algorithm}
	\caption{Natural Policy Gradient}
	\label{algo_1}
	\begin{algorithmic}[1]
	\STATE \textbf{Input:} $\eta,\alpha$: Learning rates, $J,L$: Number of execution steps\\
	\hspace{1.0cm}$\mathbf{w}_0,\Phi_0$: Initial parameters, \\
	\hspace{1.0cm}$\boldsymbol{\mu}_0$: Initial local state distribution\\
	\hspace{1.0cm}$g_0$: Initial global state
	\STATE \textbf{Initialization:} $\Phi\gets \Phi_0$ 
	
		\FOR{$j\in\{0,1,\cdots,J-1\}$}
		{
			\STATE $\mathbf{w}_{j,0}\gets \mathbf{w}_0$\\
			\FOR {$l\in\{0,1,\cdots,L-1\}$}
			{
				\STATE Sample $(x,\boldsymbol{\mu},g, u)\sim\zeta_{(\boldsymbol{\mu}_0, g_0)}^{\Phi_j}$ and $\hat{A}_{\Phi_j}(x,\boldsymbol{\mu},g,u)$ using Algorithm \ref{algo_2}\\
				\STATE Compute $\mathbf{h}_{j,l}$ using $(\ref{sub_prob_grad_update})$\\
				\STATE $\mathbf{w}_{j,l+1}\gets\mathbf{w}_{j,l}-\alpha\mathbf{h}_{j,l}$
			}
			\ENDFOR
			\STATE	$\mathbf{w}_j\gets\dfrac{1}{L}\sum_{l=1}^{L}\mathbf{w}_{j,l}$\\
			\STATE	$\Phi_{j+1}\gets \Phi_j +\eta \mathbf{w}_j$
		}
		\ENDFOR

	\STATE \textbf{Output:} $\{\Phi_1,\cdots,\Phi_J\}$: Policy parameters
		\end{algorithmic}
\end{algorithm}


Based on the result (Theorem 4.9) of \citep{liu2020improved}, in Lemma \ref{lemma_0}, we establish the convergence of Algorithm 1 and characterize its sample complexity. However, the following assumptions are needed to prove the Lemma. The assumptions stated below are respectively similar to Assumptions 2.1, 4.2, 4.4 of \citep{liu2020improved}.


\begin{assumption}
	\label{ass_6}
	$\forall \Phi\in\mathbb{R}^{\mathrm{d}}$, $\forall (\boldsymbol{\mu}_0, g_0)\in\Delta(\mathcal{X})\times \mathcal{G}$, for some $\chi >0$,  $F_{(\boldsymbol{\mu}_0, g_0)}(\Phi)-\chi I_{\mathrm{d}}$ is positive semi-definite  where $F_{(\boldsymbol{\mu}_0, g_0)}(\Phi)$ is defined as stated below.
	\begin{align*}
	F_{(\boldsymbol{\mu}_0, g_0)}(\Phi)\triangleq & \mathbb{E}_{(x,\boldsymbol{\mu},g,u)\sim \zeta_{(\boldsymbol{\mu}_0, g_0)}^{\Phi}}\Big[\left\lbrace\nabla_{\Phi}\pi_{\Phi}(x,\boldsymbol{\mu}, g)(u)\right\rbrace	\times\left\lbrace\nabla_{\Phi}\log\pi_{\Phi}(x,\boldsymbol{\mu}, g)(u)\right\rbrace^{\mathrm{T}}\Big]
	\end{align*} 
\end{assumption}

\begin{assumption}
	\label{ass_7}
	$\forall \Phi\in\mathbb{R}^{\mathrm{d}}$, $\forall \boldsymbol{\mu}\in\Delta(\mathcal{X})$, $\forall x\in\mathcal{X}$, $\forall g\in \mathcal{G}$, $\forall u\in\mathcal{U}$, the following holds
	\begin{align*}
	\left|\nabla_{\Phi}\log\pi_{\Phi}(x,\boldsymbol{\