\begin{table}[t]
\small
\centering
\setlength\tabcolsep{1.0pt}
\def1.1{1.1}
\begin{tabular}{C{3.5cm}|C{1.1cm}C{1.1cm}|C{1.1cm}C{1.1cm}}
\specialrule{.1em}{.05em}{.05em}
\multirow{2}{*}{method} & \multicolumn{2}{c|}{ ZJU-MoCap} & \multicolumn{2}{c}{AIST} \\ 
&  PSNR $\uparrow$ &  SSIM $\uparrow$ & PSNR $\uparrow$ &  SSIM $\uparrow$ \\ \hline
pixelNeRF~\cite{yu2021pixelnerf} & 22.13 & 0.8604 & 16.79 & 0.6308\\
NHP~\cite{kwon2021neural} & 24.01 & 0.8953 & 16.58 & 0.6934\\ 
\textbf{MonoNHR (Ours)} & \textbf{25.36} & \textbf{0.9093} & \textbf{17.61} & \textbf{0.7186} \\ \specialrule{.1em}{.05em}{.05em}
\end{tabular}
\vspace*{-3mm}
\caption{
{Comparison of state-of-the-art NeRF-based neural human rendering methods on ZJU-MoCap and AIST.} 
}
\label{table:sota}
\end{table}
\author{
Hongsuk Choi$^{*1,3}$
\and
Gyeongsik Moon$^{*2}$
\and
Matthieu Armando$^3$
\and
Vincent Leroy$^3$
\and
Kyoung Mu Lee$^1$
\and
Gr\'egory Rogez$^3$
\and
\\
$^1$ Dept. of ECE \& ASRI, Seoul National University, Korea
\\
$^2$ Meta Reality Labs Research
\hspace{1.1cm}
$^3$ NAVER LABS Europe
\\
\small \texttt {redstonepo@gmail.com, mks0601@fb.com, kyoungmu@snu.ac.kr}\\ \small \texttt {\{matthieu.armando,vincent.leroy,gregory.rogez\}@naverlabs.com}
}

\subsection{Backbone}
First, a backbone extracts image features that will be used to condition the neural radiance field. Similarly to~\cite{yu2021pixelnerf}, this allows MonoNHR to learn priors relevant to human rendering, which ultimately enables our method to work on subjects that were not seen during training.
We use a ResNet18~\cite{he2016deep} as backbone.
It takes a masked human image $\mathbf{I} \in \mathbb{R}^{3 \times 256 \times 256}$ as input and produces a feature map $\mathbf{F} \in \mathbb{R}^{c_p \times 128 \times 128}$, with
$c_p=64$ the feature's channel dimension. An off-the-shelf semantic part segmentation method, PGN~\cite{gong2018instance}, is used to segment the subjects in the images.
The feature map $\mathbf{F}$ is used in two ways:

\noindent\textbf{Mesh feature.} Following NHP~\cite{kwon2021neural}, we sample per-vertex image features from $\mathbf{F}$, given SMPL~\cite{loper2015smpl} mesh vertices.
We project these 3D vertices onto the image plane and use bilinear interpolation to sample the corresponding image features. 
Then, we concatenate the per-vertex image features and the vertices' root joint (~\textit{i.e.}, pelvis)-relative depths, to distinguish the features of both occluded and occluding parts.
The concatenated feature is called the \textcolor[RGB]{98,163,59}{mesh feature} and denoted as \textcolor[RGB]{98,163,59}{$\mathbf{M} \in \mathbb{R}^{M \times (c_p+1)}$}, where $M$ denotes the number of SMPL mesh vertices. We feed these to the geometry branch.

\noindent\textbf{Pixel feature.} We sample 3D query points from a ray of a target view (rendering view), following the quadrature rule of the volume rendering discussed by Max~\cite{max1995optical}. The rays are bounded by the given mesh's 3D bounding box, following NB~\cite{peng2021neural} and NHP~\cite{kwon2021neural}. 
The 3D query point $\mathbf{x}\in \mathbb{R}^3$ is equipped with appearance information from its projection on the image plane. We coin this information the \textcolor[RGB]{252,99,159}{pixel feature $\mathbf{p} \in \mathbb{R}^{c_p}$}.
The pixel feature $\mathbf{p}$ is fed to both geometry and texture branches with $\mathbf{x}$'s \textcolor[RGB]{54,93,183}{root joint-relative depth $\mathrm{z}$}.
The root joint-relative depth is defined in the input view's camera-centered coordinate system.

\begin{equation}
    \mathbf{c}(\mathbf{x}) = M_{\mathbf{c}}(\mathbf{p}, \mathrm{z}, \mathbf{R}\mathbf{d}; \sigma),
\end{equation}

where $\delta_i = || \mathbf{x}_{i + 1} - \mathbf{x}_{i} ||_2$ is the distance between adjacent sampled points, and $\mathbf{r}$ is the camera ray. 
In practice, we set $N$ to 64 following~\cite{peng2021neural,kwon2021neural}.

\begin{equation}
    L_{\text{render}} = \sum\limits_{\mathbf{r} \in \mathcal{R}} \normsq{\hat{C}(\mathbf{r}) - C(\mathbf{r})},
\end{equation}

\subsection{Comparison with state-of-the-art methods}
\noindent\textbf{Overall comparison.}
The Figure~\ref{fig:sota} and Table~\ref{table:sota} show that MonoNHR produces the best results on both ZJU-MoCap~\cite{peng2021neural} and AIST~\cite{li2021learn,tsuchida2019aist}.
{"value": "22.13", "type": "Result", "training data/set": "ZJU-MoCap", "test data/set": "ZJU-MoCap", "task": ["human rendering", "free-viewpoint renderings"], "metric": ["PSNR", "peak signal-to-noise ratio"], "experimental settings": {"xx": "yy"}, "model": "pixelNeRF", "model settings": {"xx": "yy"}} {"value": "0.8604", "type": "Result", "training data/set": "ZJU-MoCap", "test data/set": "ZJU-MoCap", "task": ["human rendering", "free-viewpoint renderings"], "metric": ["SSIM", "structural similarity index"], "experimental settings": {"xx": "yy"}, "model": "pixelNeRF", "model settings": {"xx": "yy"}} {"value": "16.79", "type": "Result", "training data/set": "AIST", "test data/set": "AIST", "task": ["human rendering", "free-viewpoint renderings"], "metric": ["PSNR", "peak signal-to-noise ratio"], "experimental settings": {"xx": "yy"}, "model": "pixelNeRF", "model settings": {"xx": "yy"}} {"value": "0.6308", "type": "Result", "training data/set": "AIST", "test data/set": "AIST", "task": ["human rendering", "free-viewpoint renderings"], "metric": ["SSIM", "structural similarity index"], "experimental settings": {"xx": "yy"}, "model": "pixelNeRF", "model settings": {"xx": "yy"}} {"value": "24.01", "type": "Result", "training data/set": "ZJU-MoCap", "test data/set": "ZJU-MoCap", "task": ["human rendering", "free-viewpoint renderings"], "metric": ["PSNR", "peak signal-to-noise ratio"], "experimental settings": {"xx": "yy"}, "model": "NHP", "model settings": {"xx": "yy"}} {"value": "0.8953", "type": "Result", "training data/set": "ZJU-MoCap", "test data/set": "ZJU-MoCap", "task": ["human rendering", "free-viewpoint renderings"], "metric": ["SSIM", "structural similarity index"], "experimental settings": {"xx": "yy"}, "model": "NHP", "model settings": {"xx": "yy"}} {"value": "16.58", "type": "Result", "training data/set": "AIST", "test data/set": "AIST", "task": ["human rendering", "free-viewpoint renderings"], "metric": ["PSNR", "peak signal-to-noise ratio"], "experimental settings": {"xx": "yy"}, "model": "NHP", "model settings": {"xx": "yy"}} {"value": "0.6934", "type": "Result", "training data/set": "AIST", "test data/set": "AIST", "task": ["human rendering", "free-viewpoint renderings"], "metric": ["SSIM", "structural similarity index"], "experimental settings": {"xx": "yy"}, "model": "NHP", "model settings": {"xx": "yy"}} {"value": "25.36", "type": "Result", "training data/set": "ZJU-MoCap", "test data/set": "ZJU-MoCap", "task": ["human rendering", "free-viewpoint renderings"], "metric": ["PSNR", "peak signal-to-noise ratio"], "experimental settings": {"xx": "yy"}, "model": "MonoNHR", "model settings": {"xx": "yy"}} {"value": "0.9093", "type": "Result", "training data/set": "ZJU-MoCap", "test data/set": "ZJU-MoCap", "task": ["human rendering", "free-viewpoint renderings"], "metric": ["SSIM", "structural similarity index"], "experimental settings": {"xx": "yy"}, "model": "MonoNHR", "model settings": {"xx": "yy"}} {"value": "17.61", "type": "Result", "training data/set": "AIST", "test data/set": "AIST", "task": ["human rendering", "free-viewpoint renderings"], "metric": ["PSNR", "peak signal-to-noise ratio"], "experimental settings": {"xx": "yy"}, "model": "MonoNHR", "model settings": {"xx": "yy"}} {"value": "0.7186", "type": "Result", "training data/set": "AIST", "test data/set": "AIST", "task": ["human rendering", "free-viewpoint renderings"], "metric": ["SSIM", "structural similarity index"], "experimental settings": {"xx": "yy"}, "model": "MonoNHR", "model settings": {"xx": "yy"}}