# -*- coding: utf-8 -*-
# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
# holder of all proprietary rights on this computer program.
# You can only use this computer program if you have closed
# a license agreement with MPG or you get the right to use the computer
# program from someone who is authorized to grant you that right.
# Any use of the computer program without a valid license is prohibited and
# liable to prosecution.
#
# Copyright©2019 Max-Planck-Gesellschaft zur Förderung
# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
# for Intelligent Systems. All rights reserved.
#
# Contact: ps-license@tuebingen.mpg.de
import logging
import os
import os.path as osp
import pickle
from collections import namedtuple
from typing import Dict, Optional, Union
import numpy as np
import torch
import torch.nn as nn
logging.getLogger("smplx").setLevel(logging.ERROR)
from .lbs import find_dynamic_lmk_idx_and_bcoords, lbs, vertices2landmarks
from .utils import (
Array,
FLAMEOutput,
MANOOutput,
SMPLHOutput,
SMPLOutput,
SMPLXOutput,
Struct,
Tensor,
find_joint_kin_chain,
to_np,
to_tensor,
)
from .vertex_ids import vertex_ids as VERTEX_IDS
from .vertex_joint_selector import VertexJointSelector
ModelOutput = namedtuple(
"ModelOutput",
[
"vertices",
"joints",
"full_pose",
"betas",
"global_orient",
"body_pose",
"expression",
"left_hand_pose",
"right_hand_pose",
"jaw_pose",
],
)
ModelOutput.__new__.__defaults__ = (None, ) * len(ModelOutput._fields)
class SMPL(nn.Module):
NUM_JOINTS = 23
NUM_BODY_JOINTS = 23
SHAPE_SPACE_DIM = 300
def __init__(
self,
model_path: str,
kid_template_path: str = "",
data_struct: Optional[Struct] = None,
create_betas: bool = True,
betas: Optional[Tensor] = None,
num_betas: int = 10,
create_global_orient: bool = True,
global_orient: Optional[Tensor] = None,
create_body_pose: bool = True,
body_pose: Optional[Tensor] = None,
create_transl: bool = True,
transl: Optional[Tensor] = None,
dtype=torch.float32,
batch_size: int = 1,
joint_mapper=None,
gender: str = "neutral",
age: str = "adult",
vertex_ids: Dict[str, int] = None,
v_template: Optional[Union[Tensor, Array]] = None,
v_personal: Optional[Union[Tensor, Array]] = None,
**kwargs,
) -> None:
"""SMPL model constructor
Parameters
----------
model_path: str
The path to the folder or to the file where the model
parameters are stored
data_struct: Strct
A struct object. If given, then the parameters of the model are
read from the object. Otherwise, the model tries to read the
parameters from the given `model_path`. (default = None)
create_global_orient: bool, optional
Flag for creating a member variable for the global orientation
of the body. (default = True)
global_orient: torch.tensor, optional, Bx3
The default value for the global orientation variable.
(default = None)
create_body_pose: bool, optional
Flag for creating a member variable for the pose of the body.
(default = True)
body_pose: torch.tensor, optional, Bx(Body Joints * 3)
The default value for the body pose variable.
(default = None)
num_betas: int, optional
Number of shape components to use
(default = 10).
create_betas: bool, optional
Flag for creating a member variable for the shape space
(default = True).
betas: torch.tensor, optional, Bx10
The default value for the shape member variable.
(default = None)
create_transl: bool, optional
Flag for creating a member variable for the translation
of the body. (default = True)
transl: torch.tensor, optional, Bx3
The default value for the transl variable.
(default = None)
dtype: torch.dtype, optional
The data type for the created variables
batch_size: int, optional
The batch size used for creating the member variables
joint_mapper: object, optional
An object that re-maps the joints. Useful if one wants to
re-order the SMPL joints to some other convention (e.g. MSCOCO)
(default = None)
gender: str, optional
Which gender to load
vertex_ids: dict, optional
A dictionary containing the indices of the extra vertices that
will be selected
"""
self.gender = gender
self.age = age
if data_struct is None:
if osp.isdir(model_path):
model_fn = "SMPL_{}.{ext}".format(gender.upper(), ext="pkl")
smpl_path = os.path.join(model_path, model_fn)
else:
smpl_path = model_path
assert osp.exists(smpl_path), "Path {} does not exist!".format(smpl_path)
with open(smpl_path, "rb") as smpl_file:
data_struct = Struct(**pickle.load(smpl_file, encoding="latin1"))
super(SMPL, self).__init__()
self.batch_size = batch_size
shapedirs = data_struct.shapedirs
if shapedirs.shape[-1] < self.SHAPE_SPACE_DIM:
# print(f'WARNING: You are using a {self.name()} model, with only'
# ' 10 shape coefficients.')
num_betas = min(num_betas, 10)
else:
num_betas = min(num_betas, self.SHAPE_SPACE_DIM)
if self.age == "kid":
v_template_smil = np.load(kid_template_path)
v_template_smil -= np.mean(v_template_smil, axis=0)
v_template_diff = np.expand_dims(v_template_smil - data_struct.v_template, axis=2)
shapedirs = np.concatenate((shapedirs[:, :, :num_betas], v_template_diff), axis=2)
num_betas = num_betas + 1
self._num_betas = num_betas
shapedirs = shapedirs[:, :, :num_betas]
# The shape components
self.register_buffer("shapedirs", to_tensor(to_np(shapedirs), dtype=dtype))
if vertex_ids is None:
# SMPL and SMPL-H share the same topology, so any extra joints can
# be drawn from the same place
vertex_ids = VERTEX_IDS["smplh"]
self.dtype = dtype
self.joint_mapper = joint_mapper
self.vertex_joint_selector = VertexJointSelector(vertex_ids=vertex_ids, **kwargs)
self.faces = data_struct.f
self.register_buffer(
"faces_tensor",
to_tensor(to_np(self.faces, dtype=np.int64), dtype=torch.long),
)
if create_betas:
if betas is None:
default_betas = torch.zeros([batch_size, self.num_betas], dtype=dtype)
else:
if torch.is_tensor(betas):
default_betas = betas.clone().detach()
else:
default_betas = torch.tensor(betas, dtype=dtype)
self.register_parameter("betas", nn.Parameter(default_betas, requires_grad=True))
# The tensor that contains the global rotation of the model
# It is separated from the pose of the joints in case we wish to
# optimize only over one of them
if create_global_orient:
if global_orient is None:
default_global_orient = torch.zeros([batch_size, 3], dtype=dtype)
else:
if torch.is_tensor(global_orient):
default_global_orient = global_orient.clone().detach()
else:
default_global_orient = torch.tensor(global_orient, dtype=dtype)
global_orient = nn.Parameter(default_global_orient, requires_grad=True)
self.register_parameter("global_orient", global_orient)
if create_body_pose:
if body_pose is None:
default_body_pose = torch.zeros([batch_size, self.NUM_BODY_JOINTS * 3], dtype=dtype)
else:
if torch.is_tensor(body_pose):
default_body_pose = body_pose.clone().detach()
else:
default_body_pose = torch.tensor(body_pose, dtype=dtype)
self.register_parameter(
"body_pose", nn.Parameter(default_body_pose, requires_grad=True)
)
if create_transl:
if transl is None:
default_transl = torch.zeros([batch_size, 3], dtype=dtype, requires_grad=True)
else:
default_transl = torch.tensor(transl, dtype=dtype)
self.register_parameter("transl", nn.Parameter(default_transl, requires_grad=True))
if v_template is None:
v_template = data_struct.v_template
if not torch.is_tensor(v_template):
v_template = to_tensor(to_np(v_template), dtype=dtype)
if v_personal is not None:
v_personal = to_tensor(to_np(v_personal), dtype=dtype)
v_template += v_personal
# The vertices of the template model
self.register_buffer("v_template", v_template)
j_regressor = to_tensor(to_np(data_struct.J_regressor), dtype=dtype)
self.register_buffer("J_regressor", j_regressor)
# Pose blend shape basis: 6890 x 3 x 207, reshaped to 6890*3 x 207
num_pose_basis = data_struct.posedirs.shape[-1]
# 207 x 20670
posedirs = np.reshape(data_struct.posedirs, [-1, num_pose_basis]).T
self.register_buffer("posedirs", to_tensor(to_np(posedirs), dtype=dtype))
# indices of parents for each joints
parents = to_tensor(to_np(data_struct.kintree_table[0])).long()
parents[0] = -1
self.register_buffer("parents", parents)
self.register_buffer("lbs_weights", to_tensor(to_np(data_struct.weights), dtype=dtype))
@property
def num_betas(self):
return self._num_betas
@property
def num_expression_coeffs(self):
return 0
def create_mean_pose(self, data_struct) -> Tensor:
pass
def name(self) -> str:
return "SMPL"
@torch.no_grad()
def reset_params(self, **params_dict) -> None:
for param_name, param in self.named_parameters():
if param_name in params_dict:
param[:] = torch.tensor(params_dict[param_name])
else:
param.fill_(0)
def get_num_verts(self) -> int:
return self.v_template.shape[0]
def get_num_faces(self) -> int:
return self.faces.shape[0]
def extra_repr(self) -> str:
msg = [
f"Gender: {self.gender.upper()}",
f"Number of joints: {self.J_regressor.shape[0]}",
f"Betas: {self.num_betas}",
]
return "\n".join(msg)
def forward(
self,
betas: Optional[Tensor] = None,
body_pose: Optional[Tensor] = None,
global_orient: Optional[Tensor] = None,
transl: Optional[Tensor] = None,
return_verts=True,
return_full_pose: bool = False,
pose2rot: bool = True,
**kwargs,
) -> SMPLOutput:
"""Forward pass for the SMPL model
Parameters
----------
global_orient: torch.tensor, optional, shape Bx3
If given, ignore the member variable and use it as the global
rotation of the body. Useful if someone wishes to predicts this
with an external model. (default=None)
betas: torch.tensor, optional, shape BxN_b
If given, ignore the member variable `betas` and use it
instead. For example, it can used if shape parameters
`betas` are predicted from some external model.
(default=None)
body_pose: torch.tensor, optional, shape Bx(J*3)
If given, ignore the member variable `body_pose` and use it
instead. For example, it can used if someone predicts the
pose of the body joints are predicted from some external model.
It should be a tensor that contains joint rotations in
axis-angle format. (default=None)
transl: torch.tensor, optional, shape Bx3
If given, ignore the member variable `transl` and use it
instead. For example, it can used if the translation
`transl` is predicted from some external model.
(default=None)
return_verts: bool, optional
Return the vertices. (default=True)
return_full_pose: bool, optional
Returns the full axis-angle pose vector (default=False)
Returns
-------
"""
# If no shape and pose parameters are passed along, then use the
# ones from the module
global_orient = (global_orient if global_orient is not None else self.global_orient)
body_pose = body_pose if body_pose is not None else self.body_pose
betas = betas if betas is not None else self.betas
apply_trans = transl is not None or hasattr(self, "transl")
if transl is None and hasattr(self, "transl"):
transl = self.transl
full_pose = torch.cat([global_orient, body_pose], dim=1)
batch_size = max(betas.shape[0], global_orient.shape[0], body_pose.shape[0])
if betas.shape[0] != batch_size:
num_repeats = int(batch_size / betas.shape[0])
betas = betas.expand(num_repeats, -1)
vertices, joints = lbs(
betas,
full_pose,
self.v_template,
self.shapedirs,
self.posedirs,
self.J_regressor,
self.parents,
self.lbs_weights,
pose2rot=pose2rot,
)
joints = self.vertex_joint_selector(vertices, joints)
# Map the joints to the current dataset
if self.joint_mapper is not None:
joints = self.joint_mapper(joints)
if apply_trans:
joints += transl.unsqueeze(dim=1)
vertices += transl.unsqueeze(dim=1)
output = SMPLOutput(
vertices=vertices if return_verts else None,
global_orient=global_orient,
body_pose=body_pose,
joints=joints,
betas=betas,
full_pose=full_pose if return_full_pose else None,
)
return output
class SMPLLayer(SMPL):
def __init__(self, *args, **kwargs) -> None:
# Just create a SMPL module without any member variables
super(SMPLLayer, self).__init__(
create_body_pose=False,
create_betas=False,
create_global_orient=False,
create_transl=False,
*args,
**kwargs,
)
def forward(
self,
betas: Optional[Tensor] = None,
body_pose: Optional[Tensor] = None,
global_orient: Optional[Tensor] = None,
transl: Optional[Tensor] = None,
return_verts=True,
return_full_pose: bool = False,
pose2rot: bool = True,
**kwargs,
) -> SMPLOutput:
"""Forward pass for the SMPL model
Parameters
----------
global_orient: torch.tensor, optional, shape Bx3x3
Global rotation of the body. Useful if someone wishes to
predicts this with an external model. It is expected to be in
rotation matrix format. (default=None)
betas: torch.tensor, optional, shape BxN_b
Shape parameters. For example, it can used if shape parameters
`betas` are predicted from some external model.
(default=None)
body_pose: torch.tensor, optional, shape BxJx3x3
Body pose. For example, it can used if someone predicts the
pose of the body joints are predicted from some external model.
It should be a tensor that contains joint rotations in
rotation matrix format. (default=None)
transl: torch.tensor, optional, shape Bx3
Translation vector of the body.
For example, it can used if the translation
`transl` is predicted from some external model.
(default=None)
return_verts: bool, optional
Return the vertices. (default=True)
return_full_pose: bool, optional
Returns the full axis-angle pose vector (default=False)
Returns
-------
"""
model_vars = [betas, global_orient, body_pose, transl]
batch_size = 1
for var in model_vars:
if var is None:
continue
batch_size = max(batch_size, len(var))
device, dtype = self.shapedirs.device, self.shapedirs.dtype
if global_orient is None:
global_orient = (
torch.eye(3, device=device,
dtype=dtype).view(1, 1, 3, 3).expand(batch_size, -1, -1, -1).contiguous()
)
if body_pose is None:
body_pose = (
torch.eye(3, device=device,
dtype=dtype).view(1, 1, 3,
3).expand(batch_size, self.NUM_BODY_JOINTS, -1,
-1).contiguous()
)
if betas is None:
betas = torch.zeros([batch_size, self.num_betas], dtype=dtype, device=device)
if transl is None:
transl = torch.zeros([batch_size, 3], dtype=dtype, device=device)
full_pose = torch.cat(
[
global_orient.reshape(-1, 1, 3, 3),
body_pose.reshape(-1, self.NUM_BODY_JOINTS, 3, 3),
],
dim=1,
)
vertices, joints = lbs(
betas,
full_pose,
self.v_template,
self.shapedirs,
self.posedirs,
self.J_regressor,
self.parents,
self.lbs_weights,
pose2rot=False,
)
joints = self.vertex_joint_selector(vertices, joints)
# Map the joints to the current dataset
if self.joint_mapper is not None:
joints = self.joint_mapper(joints)
if transl is not None:
joints += transl.unsqueeze(dim=1)
vertices += transl.unsqueeze(dim=1)
output = SMPLOutput(
vertices=vertices if return_verts else None,
global_orient=global_orient,
body_pose=body_pose,
joints=joints,
betas=betas,
full_pose=full_pose if return_full_pose else None,
)
return output
class SMPLH(SMPL):
# The hand joints are replaced by MANO
NUM_BODY_JOINTS = SMPL.NUM_JOINTS - 2
NUM_HAND_JOINTS = 15
NUM_JOINTS = NUM_BODY_JOINTS + 2 * NUM_HAND_JOINTS
def __init__(
self,
model_path,
kid_template_path: str = "",
data_struct: Optional[Struct] = None,
create_left_hand_pose: bool = True,
left_hand_pose: Optional[Tensor] = None,
create_right_hand_pose: bool = True,
right_hand_pose: Optional[Tensor] = None,
use_pca: bool = True,
num_pca_comps: int = 6,
flat_hand_mean: bool = False,
batch_size: int = 1,
gender: str = "neutral",
age: str = "adult",
dtype=torch.float32,
vertex_ids=None,
use_compressed: bool = True,
ext: str = "pkl",
**kwargs,
) -> None:
"""SMPLH model constructor
Parameters
----------
model_path: str
The path to the folder or to the file where the model
parameters are stored
data_struct: Strct
A struct object. If given, then the parameters of the model are
read from the object. Otherwise, the model tries to read the
parameters from the given `model_path`. (default = None)
create_left_hand_pose: bool, optional
Flag for creating a member variable for the pose of the left
hand. (default = True)
left_hand_pose: torch.tensor, optional, BxP
The default value for the left hand pose member variable.
(default = None)
create_right_hand_pose: bool, optional
Flag for creating a member variable for the pose of the right
hand. (default = True)
right_hand_pose: torch.tensor, optional, BxP
The default value for the right hand pose member variable.
(default = None)
num_pca_comps: int, optional
The number of PCA components to use for each hand.
(default = 6)
flat_hand_mean: bool, optional
If False, then the pose of the hand is initialized to False.
batch_size: int, optional
The batch size used for creating the member variables
gender: str, optional
Which gender to load
dtype: torch.dtype, optional
The data type for the created variables
vertex_ids: dict, optional
A dictionary containing the indices of the extra vertices that
will be selected
"""
self.num_pca_comps = num_pca_comps
# If no data structure is passed, then load the data from the given
# model folder
if data_struct is None:
# Load the model
if osp.isdir(model_path):
model_fn = "SMPLH_{}.{ext}".format(gender.upper(), ext=ext)
smplh_path = os.path.join(model_path, model_fn)
else:
smplh_path = model_path
assert osp.exists(smplh_path), "Path {} does not exist!".format(smplh_path)
if ext == "pkl":
with open(smplh_path, "rb") as smplh_file:
model_data = pickle.load(smplh_file, encoding="latin1")
elif ext == "npz":
model_data = np.load(smplh_path, allow_pickle=True)
else:
raise ValueError("Unknown extension: {}".format(ext))
data_struct = Struct(**model_data)
if vertex_ids is None:
vertex_ids = VERTEX_IDS["smplh"]
super(SMPLH, self).__init__(
model_path=model_path,
kid_template_path=kid_template_path,
data_struct=data_struct,
batch_size=batch_size,
vertex_ids=vertex_ids,
gender=gender,
age=age,
use_compressed=use_compressed,
dtype=dtype,
ext=ext,
**kwargs,
)
self.use_pca = use_pca
self.num_pca_comps = num_pca_comps
self.flat_hand_mean = flat_hand_mean
left_hand_components = data_struct.hands_componentsl[:num_pca_comps]
right_hand_components = data_struct.hands_componentsr[:num_pca_comps]
self.np_left_hand_components = left_hand_components
self.np_right_hand_components = right_hand_components
if self.use_pca:
self.register_buffer(
"left_hand_components", torch.tensor(left_hand_components, dtype=dtype)
)
self.register_buffer(
"right_hand_components",
torch.tensor(right_hand_components, dtype=dtype),
)
if self.flat_hand_mean:
left_hand_mean = np.zeros_like(data_struct.hands_meanl)
else:
left_hand_mean = data_struct.hands_meanl
if self.flat_hand_mean:
right_hand_mean = np.zeros_like(data_struct.hands_meanr)
else:
right_hand_mean = data_struct.hands_meanr
self.register_buffer("left_hand_mean", to_tensor(left_hand_mean, dtype=self.dtype))
self.register_buffer("right_hand_mean", to_tensor(right_hand_mean, dtype=self.dtype))
# Create the buffers for the pose of the left hand
hand_pose_dim = num_pca_comps if use_pca else 3 * self.NUM_HAND_JOINTS
if create_left_hand_pose:
if left_hand_pose is None:
default_lhand_pose = torch.zeros([batch_size, hand_pose_dim], dtype=dtype)
else:
default_lhand_pose = torch.tensor(left_hand_pose, dtype=dtype)
left_hand_pose_param = nn.Parameter(default_lhand_pose, requires_grad=True)
self.register_parameter("left_hand_pose", left_hand_pose_param)
if create_right_hand_pose:
if right_hand_pose is None:
default_rhand_pose = torch.zeros([batch_size, hand_pose_dim], dtype=dtype)
else:
default_rhand_pose = torch.tensor(right_hand_pose, dtype=dtype)
right_hand_pose_param = nn.Parameter(default_rhand_pose, requires_grad=True)
self.register_parameter("right_hand_pose", right_hand_pose_param)
# Create the buffer for the mean pose.
pose_mean_tensor = self.create_mean_pose(data_struct, flat_hand_mean=flat_hand_mean)
if not torch.is_tensor(pose_mean_tensor):
pose_mean_tensor = torch.tensor(pose_mean_tensor, dtype=dtype)
self.register_buffer("pose_mean", pose_mean_tensor)
def create_mean_pose(self, data_struct, flat_hand_mean=False):
# Create the array for the mean pose. If flat_hand is false, then use
# the mean that is given by the data, rather than the flat open hand
global_orient_mean = torch.zeros([3], dtype=self.dtype)
body_pose_mean = torch.zeros([self.NUM_BODY_JOINTS * 3], dtype=self.dtype)
pose_mean = torch.cat(
[
global_orient_mean,
body_pose_mean,
self.left_hand_mean,
self.right_hand_mean,
],
dim=0,
)
return pose_mean
def name(self) -> str:
return "SMPL+H"
def extra_repr(self):
msg = super(SMPLH, self).extra_repr()
msg = [msg]
if self.use_pca:
msg.append(f"Number of PCA components: {self.num_pca_comps}")
msg.append(f"Flat hand mean: {self.flat_hand_mean}")
return "\n".join(msg)
def forward(
self,
betas: Optional[Tensor] = None,
global_orient: Optional[Tensor] = None,
body_pose: Optional[Tensor] = None,
left_hand_pose: Optional[Tensor] = None,
right_hand_pose: Optional[Tensor] = None,
transl: Optional[Tensor] = None,
return_verts: bool = True,
return_full_pose: bool = False,
pose2rot: bool = True,
**kwargs,
) -> SMPLHOutput:
""""""
# If no shape and pose parameters are passed along, then use the
# ones from the module
global_orient = (global_orient if global_orient is not None else self.global_orient)
body_pose = body_pose if body_pose is not None else self.body_pose
betas = betas if betas is not None else self.betas
left_hand_pose = (left_hand_pose if left_hand_pose is not None else self.left_hand_pose)
right_hand_pose = (right_hand_pose if right_hand_pose is not None else self.right_hand_pose)
apply_trans = transl is not None or hasattr(self, "transl")
if transl is None:
if hasattr(self, "transl"):
transl = self.transl
if self.use_pca:
left_hand_pose = torch.einsum("bi,ij->bj", [left_hand_pose, self.left_hand_components])
right_hand_pose = torch.einsum(
"bi,ij->bj", [right_hand_pose, self.right_hand_components]
)
full_pose = torch.cat([global_orient, body_pose, left_hand_pose, right_hand_pose], dim=1)
full_pose += self.pose_mean
vertices, joints = lbs(
betas,
full_pose,
self.v_template,
self.shapedirs,
self.posedirs,
self.J_regressor,
self.parents,
self.lbs_weights,
pose2rot=pose2rot,
)
# Add any extra joints that might be needed
joints = self.vertex_joint_selector(vertices, joints)
if self.joint_mapper is not None:
joints = self.joint_mapper(joints)
if apply_trans:
joints += transl.unsqueeze(dim=1)
vertices += transl.unsqueeze(dim=1)
output = SMPLHOutput(
vertices=vertices if return_verts else None,
joints=joints,
betas=betas,
global_orient=global_orient,
body_pose=body_pose,
left_hand_pose=left_hand_pose,
right_hand_pose=right_hand_pose,
full_pose=full_pose if return_full_pose else None,
)
return output
class SMPLHLayer(SMPLH):
def __init__(self, *args, **kwargs) -> None:
"""SMPL+H as a layer model constructor"""
super(SMPLHLayer, self).__init__(
create_global_orient=False,
create_body_pose=False,
create_left_hand_pose=False,
create_right_hand_pose=False,
create_betas=False,
create_transl=False,
*args,
**kwargs,
)
def forward(
self,
betas: Optional[Tensor] = None,
global_orient: Optional[Tensor] = None,
body_pose: Optional[Tensor] = None,
left_hand_pose: Optional[Tensor] = None,
right_hand_pose: Optional[Tensor] = None,
transl: Optional[Tensor] = None,
return_verts: bool = True,
return_full_pose: bool = False,
pose2rot: bool = True,
**kwargs,
) -> SMPLHOutput:
"""Forward pass for the SMPL+H model
Parameters
----------
global_orient: torch.tensor, optional, shape Bx3x3
Global rotation of the body. Useful if someone wishes to
predicts this with an external model. It is expected to be in
rotation matrix format. (default=None)
betas: torch.tensor, optional, shape BxN_b
Shape parameters. For example, it can used if shape parameters
`betas` are predicted from some external model.
(default=None)
body_pose: torch.tensor, optional, shape BxJx3x3
If given, ignore the member variable `body_pose` and use it
instead. For example, it can used if someone predicts the
pose of the body joints are predicted from some external model.
It should be a tensor that contains joint rotations in
rotation matrix format. (default=None)
left_hand_pose: torch.tensor, optional, shape Bx15x3x3
If given, contains the pose of the left hand.
It should be a tensor that contains joint rotations in
rotation matrix format. (default=None)
right_hand_pose: torch.tensor, optional, shape Bx15x3x3
If given, contains the pose of the right hand.
It should be a tensor that contains joint rotations in
rotation matrix format. (default=None)
transl: torch.tensor, optional, shape Bx3
Translation vector of the body.
For example, it can used if the translation
`transl` is predicted from some external model.
(default=None)
return_verts: bool, optional
Return the vertices. (default=True)
return_full_pose: bool, optional
Returns the full axis-angle pose vector (default=False)
Returns
-------
"""
model_vars = [
betas,
global_orient,
body_pose,
transl,
left_hand_pose,
right_hand_pose,
]
batch_size = 1
for var in model_vars:
if var is None:
continue
batch_size = max(batch_size, len(var))
device, dtype = self.shapedirs.device, self.shapedirs.dtype
if global_orient is None:
global_orient = (
torch.eye(3, device=device,
dtype=dtype).view(1, 1, 3, 3).expand(batch_size, -1, -1, -1).contiguous()
)
if body_pose is None:
body_pose = (
torch.eye(3, device=device,
dtype=dtype).view(1, 1, 3, 3).expand(batch_size, 21, -1, -1).contiguous()
)
if left_hand_pose is None:
left_hand_pose = (
torch.eye(3, device=device,
dtype=dtype).view(1, 1, 3, 3).expand(batch_size, 15, -1, -1).contiguous()
)
if right_hand_pose is None:
right_hand_pose = (
torch.eye(3, device=device,
dtype=dtype).view(1, 1, 3, 3).expand(batch_size, 15, -1, -1).contiguous()
)
if betas is None:
betas = torch.zeros([batch_size, self.num_betas], dtype=dtype, device=device)
if transl is None:
transl = torch.zeros([batch_size, 3], dtype=dtype, device=device)
# Concatenate all pose vectors
full_pose = torch.cat(
[
global_orient.reshape(-1, 1, 3, 3),
body_pose.reshape(-1, self.NUM_BODY_JOINTS, 3, 3),
left_hand_pose.reshape(-1, self.NUM_HAND_JOINTS, 3, 3),
right_hand_pose.reshape(-1, self.NUM_HAND_JOINTS, 3, 3),
],
dim=1,
)
vertices, joints = lbs(
betas,
full_pose,
self.v_template,
self.shapedirs,
self.posedirs,
self.J_regressor,
self.parents,
self.lbs_weights,
pose2rot=False,
)
# Add any extra joints that might be needed
joints = self.vertex_joint_selector(vertices, joints)
if self.joint_mapper is not None:
joints = self.joint_mapper(joints)
if transl is not None:
joints += transl.unsqueeze(dim=1)
vertices += transl.unsqueeze(dim=1)
output = SMPLHOutput(
vertices=vertices if return_verts else None,
joints=joints,
betas=betas,
global_orient=global_orient,
body_pose=body_pose,
left_hand_pose=left_hand_pose,
right_hand_pose=right_hand_pose,
full_pose=full_pose if return_full_pose else None,
)
return output
class SMPLX(SMPLH):
"""
SMPL-X (SMPL eXpressive) is a unified body model, with shape parameters
trained jointly for the face, hands and body.
SMPL-X uses standard vertex based linear blend skinning with learned
corrective blend shapes, has N=10475 vertices and K=54 joints,
which includes joints for the neck, jaw, eyeballs and fingers.
"""
NUM_BODY_JOINTS = SMPLH.NUM_BODY_JOINTS # 21
NUM_HAND_JOINTS = 15
NUM_FACE_JOINTS = 3
NUM_JOINTS = NUM_BODY_JOINTS + 2 * NUM_HAND_JOINTS + NUM_FACE_JOINTS
EXPRESSION_SPACE_DIM = 100
NECK_IDX = 12
def __init__(
self,
model_path: str,
kid_template_path: str = "",
num_expression_coeffs: int = 10,
create_expression: bool = True,
expression: Optional[Tensor] = None,
create_jaw_pose: bool = True,
jaw_pose: Optional[Tensor] = None,
create_leye_pose: bool = True,
leye_pose: Optional[Tensor] = None,
create_reye_pose=True,
reye_pose: Optional[Tensor] = None,
use_face_contour: bool = False,
batch_size: int = 1,
gender: str = "neutral",
age: str = "adult",
dtype=torch.float32,
ext: str = "npz",
**kwargs,
) -> None:
"""SMPLX model constructor
Parameters
----------
model_path: str
The path to the folder or to the file where the model
parameters are stored
num_expression_coeffs: int, optional
Number of expression components to use
(default = 10).
create_expression: bool, optional
Flag for creating a member variable for the expression space
(default = True).
expression: torch.tensor, optional, Bx10
The default value for the expression member variable.
(default = None)
create_jaw_pose: bool, optional
Flag for creating a member variable for the jaw pose.
(default = False)
jaw_pose: torch.tensor, optional, Bx3
The default value for the jaw pose variable.
(default = None)
create_leye_pose: bool, optional
Flag for creating a member variable for the left eye pose.
(default = False)
leye_pose: torch.tensor, optional, Bx10
The default value for the left eye pose variable.
(default = None)
create_reye_pose: bool, optional
Flag for creating a member variable for the right eye pose.
(default = False)
reye_pose: torch.tensor, optional, Bx10
The default value for the right eye pose variable.
(default = None)
use_face_contour: bool, optional
Whether to compute the keypoints that form the facial contour
batch_size: int, optional
The batch size used for creating the member variables
gender: str, optional
Which gender to load
dtype: torch.dtype
The data type for the created variables
"""
# Load the model
from huggingface_hub import hf_hub_download
model_fn = "SMPLX_{}.{ext}".format(gender.upper(), ext=ext)
smplx_path = hf_hub_download(
repo_id=model_path, use_auth_token=os.environ["ICON"], filename=f"models/{model_fn}"
)
if ext == "pkl":
with open(smplx_path, "rb") as smplx_file:
model_data = pickle.load(smplx_file, encoding="latin1")
elif ext == "npz":
model_data = np.load(smplx_path, allow_pickle=True)
else:
raise ValueError("Unknown extension: {}".format(ext))
data_struct = Struct(**model_data)
super(SMPLX, self).__init__(
model_path=model_path,
kid_template_path=kid_template_path,
data_struct=data_struct,
dtype=dtype,
batch_size=batch_size,
vertex_ids=VERTEX_IDS["smplx"],
gender=gender,
age=age,
ext=ext,
**kwargs,
)
lmk_faces_idx = data_struct.lmk_faces_idx
self.register_buffer("lmk_faces_idx", torch.tensor(lmk_faces_idx, dtype=torch.long))
lmk_bary_coords = data_struct.lmk_bary_coords
self.register_buffer("lmk_bary_coords", torch.tensor(lmk_bary_coords, dtype=dtype))
self.use_face_contour = use_face_contour
if self.use_face_contour:
dynamic_lmk_faces_idx = data_struct.dynamic_lmk_faces_idx
dynamic_lmk_faces_idx = torch.tensor(dynamic_lmk_faces_idx, dtype=torch.long)
self.register_buffer("dynamic_lmk_faces_idx", dynamic_lmk_faces_idx)
dynamic_lmk_bary_coords = data_struct.dynamic_lmk_bary_coords
dynamic_lmk_bary_coords = torch.tensor(dynamic_lmk_bary_coords, dtype=dtype)
self.register_buffer("dynamic_lmk_bary_coords", dynamic_lmk_bary_coords)
neck_kin_chain = find_joint_kin_chain(self.NECK_IDX, self.parents)
self.register_buffer("neck_kin_chain", torch.tensor(neck_kin_chain, dtype=torch.long))
if create_jaw_pose:
if jaw_pose is None:
default_jaw_pose = torch.zeros([batch_size, 3], dtype=dtype)
else:
default_jaw_pose = torch.tensor(jaw_pose, dtype=dtype)
jaw_pose_param = nn.Parameter(default_jaw_pose, requires_grad=True)
self.register_parameter("jaw_pose", jaw_pose_param)
if create_leye_pose:
if leye_pose is None:
default_leye_pose = torch.zeros([batch_size, 3], dtype=dtype)
else:
default_leye_pose = torch.tensor(leye_pose, dtype=dtype)
leye_pose_param = nn.Parameter(default_leye_pose, requires_grad=True)
self.register_parameter("leye_pose", leye_pose_param)
if create_reye_pose:
if reye_pose is None:
default_reye_pose = torch.zeros([batch_size, 3], dtype=dtype)
else:
default_reye_pose = torch.tensor(reye_pose, dtype=dtype)
reye_pose_param = nn.Parameter(default_reye_pose, requires_grad=True)
self.register_parameter("reye_pose", reye_pose_param)
shapedirs = data_struct.shapedirs
if len(shapedirs.shape) < 3:
shapedirs = shapedirs[:, :, None]
if shapedirs.shape[-1] < self.SHAPE_SPACE_DIM + self.EXPRESSION_SPACE_DIM:
# print(f'WARNING: You are using a {self.name()} model, with only'
# ' 10 shape and 10 expression coefficients.')
expr_start_idx = 10
expr_end_idx = 20
num_expression_coeffs = min(num_expression_coeffs, 10)
else:
expr_start_idx = self.SHAPE_SPACE_DIM
expr_end_idx = self.SHAPE_SPACE_DIM + num_expression_coeffs
num_expression_coeffs = min(num_expression_coeffs, self.EXPRESSION_SPACE_DIM)
self._num_expression_coeffs = num_expression_coeffs
expr_dirs = shapedirs[:, :, expr_start_idx:expr_end_idx]
self.register_buffer("expr_dirs", to_tensor(to_np(expr_dirs), dtype=dtype))
if create_expression:
if expression is None:
default_expression = torch.zeros([batch_size, self.num_expression_coeffs],
dtype=dtype)
else:
default_expression = torch.tensor(expression, dtype=dtype)
expression_param = nn.Parameter(default_expression, requires_grad=True)
self.register_parameter("expression", expression_param)
def name(self) -> str:
return "SMPL-X"
@property
def num_expression_coeffs(self):
return self._num_expression_coeffs
def create_mean_pose(self, data_struct, flat_hand_mean=False):
# Create the array for the mean pose. If flat_hand is false, then use
# the mean that is given by the data, rather than the flat open hand
global_orient_mean = torch.zeros([3], dtype=self.dtype)
body_pose_mean = torch.zeros([self.NUM_BODY_JOINTS * 3], dtype=self.dtype)
jaw_pose_mean = torch.zeros([3], dtype=self.dtype)
leye_pose_mean = torch.zeros([3], dtype=self.dtype)
reye_pose_mean = torch.zeros([3], dtype=self.dtype)
pose_mean = np.concatenate(
[
global_orient_mean,
body_pose_mean,
jaw_pose_mean,
leye_pose_mean,
reye_pose_mean,
self.left_hand_mean,
self.right_hand_mean,
],
axis=0,
)
return pose_mean
def extra_repr(self):
msg = super(SMPLX, self).extra_repr()
msg = [msg, f"Number of Expression Coefficients: {self.num_expression_coeffs}"]
return "\n".join(msg)
def forward(
self,
betas: Optional[Tensor] = None,
global_orient: Optional[Tensor] = None,
body_pose: Optional[Tensor] = None,
left_hand_pose: Optional[Tensor] = None,
right_hand_pose: Optional[Tensor] = None,
transl: Optional[Tensor] = None,
expression: Optional[Tensor] = None,
jaw_pose: Optional[Tensor] = None,
leye_pose: Optional[Tensor] = None,
reye_pose: Optional[Tensor] = None,
return_verts: bool = True,
return_full_pose: bool = False,
pose2rot: bool = True,
return_joint_transformation: bool = False,
return_vertex_transformation: bool = False,
pose_type: str = 'posed',
**kwargs,
) -> SMPLXOutput:
"""
Forward pass for the SMPLX model
Parameters
----------
global_orient: torch.tensor, optional, shape Bx3
If given, ignore the member variable and use it as the global
rotation of the body. Useful if someone wishes to predicts this
with an external model. (default=None)
betas: torch.tensor, optional, shape BxN_b
If given, ignore the member variable `betas` and use it
instead. For example, it can used if shape parameters
`betas` are predicted from some external model.
(default=None)
expression: torch.tensor, optional, shape BxN_e
If given, ignore the member variable `expression` and use it
instead. For example, it can used if expression parameters
`expression` are predicted from some external model.
body_pose: torch.tensor, optional, shape Bx(J*3)
If given, ignore the member variable `body_pose` and use it
instead. For example, it can used if someone predicts the
pose of the body joints are predicted from some external model.
It should be a tensor that contains joint rotations in
axis-angle format. (default=None)
left_hand_pose: torch.tensor, optional, shape BxP
If given, ignore the member variable `left_hand_pose` and
use this instead. It should either contain PCA coefficients or
joint rotations in axis-angle format.
right_hand_pose: torch.tensor, optional, shape BxP
If given, ignore the member variable `right_hand_pose` and
use this instead. It should either contain PCA coefficients or
joint rotations in axis-angle format.
jaw_pose: torch.tensor, optional, shape Bx3
If given, ignore the member variable `jaw_pose` and
use this instead. It should either joint rotations in
axis-angle format.
transl: torch.tensor, optional, shape Bx3
If given, ignore the member variable `transl` and use it
instead. For example, it can used if the translation
`transl` is predicted from some external model.
(default=None)
return_verts: bool, optional
Return the vertices. (default=True)
return_full_pose: bool, optional
Returns the full axis-angle pose vector (default=False)
Returns
-------
output: ModelOutput
A named tuple of type `ModelOutput`
"""
# If no shape and pose parameters are passed along, then use the
# ones from the module
global_orient = (global_orient if global_orient is not None else self.global_orient)
body_pose = body_pose if body_pose is not None else self.body_pose
betas = betas if betas is not None else self.betas
left_hand_pose = (left_hand_pose if left_hand_pose is not None else self.left_hand_pose)
right_hand_pose = (right_hand_pose if right_hand_pose is not None else self.right_hand_pose)
jaw_pose = jaw_pose if jaw_pose is not None else self.jaw_pose
leye_pose = leye_pose if leye_pose is not None else self.leye_pose
reye_pose = reye_pose if reye_pose is not None else self.reye_pose
expression = expression if expression is not None else self.expression
apply_trans = transl is not None or hasattr(self, "transl")
if transl is None:
if hasattr(self, "transl"):
transl = self.transl
if self.use_pca:
left_hand_pose = torch.einsum("bi,ij->bj", [left_hand_pose, self.left_hand_components])
right_hand_pose = torch.einsum(
"bi,ij->bj", [right_hand_pose, self.right_hand_components]
)
full_pose = torch.cat(
[
global_orient,
body_pose,
jaw_pose,
leye_pose,
reye_pose,
left_hand_pose,
right_hand_pose,
],
dim=1,
)
if pose_type == "t-pose":
full_pose *= 0.0
elif pose_type == "a-pose":
body_pose = torch.zeros_like(body_pose).view(body_pose.shape[0], -1, 3)
body_pose[:, 15] = torch.tensor([0., 0., -45 * np.pi / 180.])
body_pose[:, 16] = torch.tensor([0., 0., 45 * np.pi / 180.])
body_pose = body_pose.view(body_pose.shape[0], -1)
full_pose = torch.cat(
[
global_orient * 0.,
body_pose,
jaw_pose * 0.,
leye_pose * 0.,
reye_pose * 0.,
left_hand_pose * 0.,
right_hand_pose * 0.,
],
dim=1,
)
elif pose_type == "da-pose":
body_pose = torch.zeros_like(body_pose).view(body_pose.shape[0], -1, 3)
body_pose[:, 0] = torch.tensor([0., 0., 30 * np.pi / 180.])
body_pose[:, 1] = torch.tensor([0., 0., -30 * np.pi / 180.])
body_pose = body_pose.view(body_pose.shape[0], -1)
full_pose = torch.cat(
[
global_orient * 0.,
body_pose,
jaw_pose * 0.,
leye_pose * 0.,
reye_pose * 0.,
left_hand_pose * 0.,
right_hand_pose * 0.,
],
dim=1,
)
# Add the mean pose of the model. Does not affect the body, only the
# hands when flat_hand_mean == False
# full_pose += self.pose_mean
batch_size = max(betas.shape[0], global_orient.shape[0], body_pose.shape[0])
# Concatenate the shape and expression coefficients
scale = int(batch_size / betas.shape[0])
if scale > 1:
betas = betas.expand(scale, -1)
shape_components = torch.cat([betas, expression], dim=-1)
shapedirs = torch.cat([self.shapedirs, self.expr_dirs], dim=-1)
if return_joint_transformation or return_vertex_transformation:
vertices, joints, joint_transformation, vertex_transformation = lbs(
shape_components,
full_pose,
self.v_template,
shapedirs,
self.posedirs,
self.J_regressor,
self.parents,
self.lbs_weights,
pose2rot=pose2rot,
return_transformation=True,
)
else:
vertices, joints = lbs(
shape_components,
full_pose,
self.v_template,
shapedirs,
self.posedirs,
self.J_regressor,
self.parents,
self.lbs_weights,
pose2rot=pose2rot,
)
lmk_faces_idx = (self.lmk_faces_idx.unsqueeze(dim=0).expand(batch_size, -1).contiguous())
lmk_bary_coords = self.lmk_bary_coords.unsqueeze(dim=0).repeat(self.batch_size, 1, 1)
if self.use_face_contour:
lmk_idx_and_bcoords = find_dynamic_lmk_idx_and_bcoords(
vertices,
full_pose,
self.dynamic_lmk_faces_idx,
self.dynamic_lmk_bary_coords,
self.neck_kin_chain,
pose2rot=True,
)
dyn_lmk_faces_idx, dyn_lmk_bary_coords = lmk_idx_and_bcoords
lmk_faces_idx = torch.cat([lmk_faces_idx, dyn_lmk_faces_idx], 1)
lmk_bary_coords = torch.cat([
lmk_bary_coords.expand(batch_size, -1, -1), dyn_lmk_bary_coords
], 1)
landmarks = vertices2landmarks(vertices, self.faces_tensor, lmk_faces_idx, lmk_bary_coords)
# Add any extra joints that might be needed
joints = self.vertex_joint_selector(vertices, joints)
# Add the landmarks to the joints
joints = torch.cat([joints, landmarks], dim=1)
# Map the joints to the current dataset
if self.joint_mapper is not None:
joints = self.joint_mapper(joints=joints, vertices=vertices)
if apply_trans:
joints += transl.unsqueeze(dim=1)
vertices += transl.unsqueeze(dim=1)
output = SMPLXOutput(
vertices=vertices if return_verts else None,
joints=joints,
betas=betas,
expression=expression,
global_orient=global_orient,
body_pose=body_pose,
left_hand_pose=left_hand_pose,
right_hand_pose=right_hand_pose,
jaw_pose=jaw_pose,
full_pose=full_pose if return_full_pose else None,
joint_transformation=joint_transformation if return_joint_transformation else None,
vertex_transformation=vertex_transformation if return_vertex_transformation else None,
)
return output
class SMPLXLayer(SMPLX):
def __init__(self, *args, **kwargs) -> None:
# Just create a SMPLX module without any member variables
super(SMPLXLayer, self).__init__(
create_global_orient=False,
create_body_pose=False,
create_left_hand_pose=False,
create_right_hand_pose=False,
create_jaw_pose=False,
create_leye_pose=False,
create_reye_pose=False,
create_betas=False,
create_expression=False,
create_transl=False,
*args,
**kwargs,
)
def forward(
self,
betas: Optional[Tensor] = None,
global_orient: Optional[Tensor] = None,
body_pose: Optional[Tensor] = None,
left_hand_pose: Optional[Tensor] = None,
right_hand_pose: Optional[Tensor] = None,
transl: Optional[Tensor] = None,
expression: Optional[Tensor] = None,
jaw_pose: Optional[Tensor] = None,
leye_pose: Optional[Tensor] = None,
reye_pose: Optional[Tensor] = None,
return_verts: bool = True,
return_full_pose: bool = False,
**kwargs,
) -> SMPLXOutput:
"""
Forward pass for the SMPLX model
Parameters
----------
global_orient: torch.tensor, optional, shape Bx3x3
If given, ignore the member variable and use it as the global
rotation of the body. Useful if someone wishes to predicts this
with an external model. It is expected to be in rotation matrix
format. (default=None)
betas: torch.tensor, optional, shape BxN_b
If given, ignore the member variable `betas` and use it
instead. For example, it can used if shape parameters
`betas` are predicted from some external model.
(default=None)
expression: torch.tensor, optional, shape BxN_e
Expression coefficients.
For example, it can used if expression parameters
`expression` are predicted from some external model.
body_pose: torch.tensor, optional, shape BxJx3x3
If given, ignore the member variable `body_pose` and use it
instead. For example, it can used if someone predicts the
pose of the body joints are predicted from some external model.
It should be a tensor that contains joint rotations in
rotation matrix format. (default=None)
left_hand_pose: torch.tensor, optional, shape Bx15x3x3
If given, contains the pose of the left hand.
It should be a tensor that contains joint rotations in
rotation matrix format. (default=None)
right_hand_pose: torch.tensor, optional, shape Bx15x3x3
If given, contains the pose of the right hand.
It should be a tensor that contains joint rotations in
rotation matrix format. (default=None)
jaw_pose: torch.tensor, optional, shape Bx3x3
Jaw pose. It should either joint rotations in
rotation matrix format.
transl: torch.tensor, optional, shape Bx3
Translation vector of the body.
For example, it can used if the translation
`transl` is predicted from some external model.
(default=None)
return_verts: bool, optional
Return the vertices. (default=True)
return_full_pose: bool, optional
Returns the full pose vector (default=False)
Returns
-------
output: ModelOutput
A data class that contains the posed vertices and joints
"""
device, dtype = self.shapedirs.device, self.shapedirs.dtype
model_vars = [
betas,
global_orient,
body_pose,
transl,
expression,
left_hand_pose,
right_hand_pose,
jaw_pose,
]
batch_size = 1
for var in model_vars:
if var is None:
continue
batch_size = max(batch_size, len(var))
if global_orient is None:
global_orient = (
torch.eye(3, device=device,
dtype=dtype).view(1, 1, 3, 3).expand(batch_size, -1, -1, -1).contiguous()
)
if body_pose is None:
body_pose = (
torch.eye(3, device=device,
dtype=dtype).view(1, 1, 3,
3).expand(batch_size, self.NUM_BODY_JOINTS, -1,
-1).contiguous()
)
if left_hand_pose is None:
left_hand_pose = (
torch.eye(3, device=device,
dtype=dtype).view(1, 1, 3, 3).expand(batch_size, 15, -1, -1).contiguous()
)
if right_hand_pose is None:
right_hand_pose = (
torch.eye(3, device=device,
dtype=dtype).view(1, 1, 3, 3).expand(batch_size, 15, -1, -1).contiguous()
)
if jaw_pose is None:
jaw_pose = (
torch.eye(3, device=device,
dtype=dtype).view(1, 1, 3, 3).expand(batch_size, -1, -1, -1).contiguous()
)
if leye_pose is None:
leye_pose = (
torch.eye(3, device=device,
dtype=dtype).view(1, 1, 3, 3).expand(batch_size, -1, -1, -1).contiguous()
)
if reye_pose is None:
reye_pose = (
torch.eye(3, device=device,
dtype=dtype).view(1, 1, 3, 3).expand(batch_size, -1, -1, -1).contiguous()
)
if expression is None:
expression = torch.zeros([batch_size, self.num_expression_coeffs],
dtype=dtype,
device=device)
if betas is None:
betas = torch.zeros([batch_size, self.num_betas], dtype=dtype, device=device)
if transl is None:
transl = torch.zeros([batch_size, 3], dtype=dtype, device=device)
# Concatenate all pose vectors
full_pose = torch.cat(
[
global_orient.reshape(-1, 1, 3, 3),
body_pose.reshape(-1, self.NUM_BODY_JOINTS, 3, 3),
jaw_pose.reshape(-1, 1, 3, 3),
leye_pose.reshape(-1, 1, 3, 3),
reye_pose.reshape(-1, 1, 3, 3),
left_hand_pose.reshape(-1, self.NUM_HAND_JOINTS, 3, 3),
right_hand_pose.reshape(-1, self.NUM_HAND_JOINTS, 3, 3),
],
dim=1,
)
shape_components = torch.cat([betas, expression], dim=-1)
shapedirs = torch.cat([self.shapedirs, self.expr_dirs], dim=-1)
vertices, joints = lbs(
shape_components,
full_pose,
self.v_template,
shapedirs,
self.posedirs,
self.J_regressor,
self.parents,
self.lbs_weights,
pose2rot=False,
)
lmk_faces_idx = (self.lmk_faces_idx.unsqueeze(dim=0).expand(batch_size, -1).contiguous())
lmk_bary_coords = self.lmk_bary_coords.unsqueeze(dim=0).repeat(batch_size, 1, 1)
if self.use_face_contour:
lmk_idx_and_bcoords = find_dynamic_lmk_idx_and_bcoords(
vertices,
full_pose,
self.dynamic_lmk_faces_idx,
self.dynamic_lmk_bary_coords,
self.neck_kin_chain,
pose2rot=False,
)
dyn_lmk_faces_idx, dyn_lmk_bary_coords = lmk_idx_and_bcoords
lmk_faces_idx = torch.cat([lmk_faces_idx, dyn_lmk_faces_idx], 1)
lmk_bary_coords = torch.cat([
lmk_bary_coords.expand(batch_size, -1, -1), dyn_lmk_bary_coords
], 1)
landmarks = vertices2landmarks(vertices, self.faces_tensor, lmk_faces_idx, lmk_bary_coords)
# Add any extra joints that might be needed
joints = self.vertex_joint_selector(vertices, joints)
# Add the landmarks to the joints
joints = torch.cat([joints, landmarks], dim=1)
# Map the joints to the current dataset
if self.joint_mapper is not None:
joints = self.joint_mapper(joints=joints, vertices=vertices)
if transl is not None:
joints += transl.unsqueeze(dim=1)
vertices += transl.unsqueeze(dim=1)
output = SMPLXOutput(
vertices=vertices if return_verts else None,
joints=joints,
betas=betas,
expression=expression,
global_orient=global_orient,
body_pose=body_pose,
left_hand_pose=left_hand_pose,
right_hand_pose=right_hand_pose,
jaw_pose=jaw_pose,
transl=transl,
full_pose=full_pose if return_full_pose else None,
)
return output
class MANO(SMPL):
# The hand joints are replaced by MANO
NUM_BODY_JOINTS = 1
NUM_HAND_JOINTS = 15
NUM_JOINTS = NUM_BODY_JOINTS + NUM_HAND_JOINTS
def __init__(
self,
model_path: str,
is_rhand: bool = True,
data_struct: Optional[Struct] = None,
create_hand_pose: bool = True,
hand_pose: Optional[Tensor] = None,
use_pca: bool = True,
num_pca_comps: int = 6,
flat_hand_mean: bool = False,
batch_size: int = 1,
dtype=torch.float32,
vertex_ids=None,
use_compressed: bool = True,
ext: str = "pkl",
**kwargs,
) -> None:
"""MANO model constructor
Parameters
----------
model_path: str
The path to the folder or to the file where the model
parameters are stored
data_struct: Strct
A struct object. If given, then the parameters of the model are
read from the object. Otherwise, the model tries to read the
parameters from the given `model_path`. (default = None)
create_hand_pose: bool, optional
Flag for creating a member variable for the pose of the right
hand. (default = True)
hand_pose: torch.tensor, optional, BxP
The default value for the right hand pose member variable.
(default = None)
num_pca_comps: int, optional
The number of PCA components to use for each hand.
(default = 6)
flat_hand_mean: bool, optional
If False, then the pose of the hand is initialized to False.
batch_size: int, optional
The batch size used for creating the member variables
dtype: torch.dtype, optional
The data type for the created variables
vertex_ids: dict, optional
A dictionary containing the indices of the extra vertices that
will be selected
"""
self.num_pca_comps = num_pca_comps
self.is_rhand = is_rhand
# If no data structure is passed, then load the data from the given
# model folder
if data_struct is None:
# Load the model
if osp.isdir(model_path):
model_fn = "MANO_{}.{ext}".format("RIGHT" if is_rhand else "LEFT", ext=ext)
mano_path = os.path.join(model_path, model_fn)
else:
mano_path = model_path
self.is_rhand = (True if "RIGHT" in os.path.basename(model_path) else False)
assert osp.exists(mano_path), "Path {} does not exist!".format(mano_path)
if ext == "pkl":
with open(mano_path, "rb") as mano_file:
model_data = pickle.load(mano_file, encoding="latin1")
elif ext == "npz":
model_data = np.load(mano_path, allow_pickle=True)
else:
raise ValueError("Unknown extension: {}".format(ext))
data_struct = Struct(**model_data)
if vertex_ids is None:
vertex_ids = VERTEX_IDS["smplh"]
super(MANO, self).__init__(
model_path=model_path,
data_struct=data_struct,
batch_size=batch_size,
vertex_ids=vertex_ids,
use_compressed=use_compressed,
dtype=dtype,
ext=ext,
**kwargs,
)
# add only MANO tips to the extra joints
self.vertex_joint_selector.extra_joints_idxs = to_tensor(
list(VERTEX_IDS["mano"].values()), dtype=torch.long
)
self.use_pca = use_pca
self.num_pca_comps = num_pca_comps
if self.num_pca_comps == 45:
self.use_pca = False
self.flat_hand_mean = flat_hand_mean
hand_components = data_struct.hands_components[:num_pca_comps]
self.np_hand_components = hand_components
if self.use_pca:
self.register_buffer("hand_components", torch.tensor(hand_components, dtype=dtype))
if self.flat_hand_mean:
hand_mean = np.zeros_like(data_struct.hands_mean)
else:
hand_mean = data_struct.hands_mean
self.register_buffer("hand_mean", to_tensor(hand_mean, dtype=self.dtype))
# Create the buffers for the pose of the left hand
hand_pose_dim = num_pca_comps if use_pca else 3 * self.NUM_HAND_JOINTS
if create_hand_pose:
if hand_pose is None:
default_hand_pose = torch.zeros([batch_size, hand_pose_dim], dtype=dtype)
else:
default_hand_pose = torch.tensor(hand_pose, dtype=dtype)
hand_pose_param = nn.Parameter(default_hand_pose, requires_grad=True)
self.register_parameter("hand_pose", hand_pose_param)
# Create the buffer for the mean pose.
pose_mean = self.create_mean_pose(data_struct, flat_hand_mean=flat_hand_mean)
pose_mean_tensor = pose_mean.clone().to(dtype)
# pose_mean_tensor = torch.tensor(pose_mean, dtype=dtype)
self.register_buffer("pose_mean", pose_mean_tensor)
def name(self) -> str:
return "MANO"
def create_mean_pose(self, data_struct, flat_hand_mean=False):
# Create the array for the mean pose. If flat_hand is false, then use
# the mean that is given by the data, rather than the flat open hand
global_orient_mean = torch.zeros([3], dtype=self.dtype)
pose_mean = torch.cat([global_orient_mean, self.hand_mean], dim=0)
return pose_mean
def extra_repr(self):
msg = [super(MANO, self).extra_repr()]
if self.use_pca:
msg.append(f"Number of PCA components: {self.num_pca_comps}")
msg.append(f"Flat hand mean: {self.flat_hand_mean}")
return "\n".join(msg)
def forward(
self,
betas: Optional[Tensor] = None,
global_orient: Optional[Tensor] = None,
hand_pose: Optional[Tensor] = None,
transl: Optional[Tensor] = None,
return_verts: bool = True,
return_full_pose: bool = False,
**kwargs,
) -> MANOOutput:
"""Forward pass for the MANO model"""
# If no shape and pose parameters are passed along, then use the
# ones from the module
global_orient = (global_orient if global_orient is not None else self.global_orient)
betas = betas if betas is not None else self.betas
hand_pose = hand_pose if hand_pose is not None else self.hand_pose
apply_trans = transl is not None or hasattr(self, "transl")
if transl is None:
if hasattr(self, "transl"):
transl = self.transl
if self.use_pca:
hand_pose = torch.einsum("bi,ij->bj", [hand_pose, self.hand_components])
full_pose = torch.cat([global_orient, hand_pose], dim=1)
full_pose += self.pose_mean
vertices, joints = lbs(
betas,
full_pose,
self.v_template,
self.shapedirs,
self.posedirs,
self.J_regressor,
self.parents,
self.lbs_weights,
pose2rot=True,
)
# # Add pre-selected extra joints that might be needed
# joints = self.vertex_joint_selector(vertices, joints)
if self.joint_mapper is not None:
joints = self.joint_mapper(joints)
if apply_trans:
joints = joints + transl.unsqueeze(dim=1)
vertices = vertices + transl.unsqueeze(dim=1)
output = MANOOutput(
vertices=vertices if return_verts else None,
joints=joints if return_verts else None,
betas=betas,
global_orient=global_orient,
hand_pose=hand_pose,
full_pose=full_pose if return_full_pose else None,
)
return output
class MANOLayer(MANO):
def __init__(self, *args, **kwargs) -> None:
"""MANO as a layer model constructor"""
super(MANOLayer, self).__init__(
create_global_orient=False,
create_hand_pose=False,
create_betas=False,
create_transl=False,
*args,
**kwargs,
)
def name(self) -> str:
return "MANO"
def forward(
self,
betas: Optional[Tensor] = None,
global_orient: Optional[Tensor] = None,
hand_pose: Optional[Tensor] = None,
transl: Optional[Tensor] = None,
return_verts: bool = True,
return_full_pose: bool = False,
**kwargs,
) -> MANOOutput:
"""Forward pass for the MANO model"""
device, dtype = self.shapedirs.device, self.shapedirs.dtype
if global_orient is None:
batch_size = 1
global_orient = (
torch.eye(3, device=device,
dtype=dtype).view(1, 1, 3, 3).expand(batch_size, -1, -1, -1).contiguous()
)
else:
batch_size = global_orient.shape[0]
if hand_pose is None:
hand_pose = (
torch.eye(3, device=device,
dtype=dtype).view(1, 1, 3, 3).expand(batch_size, 15, -1, -1).contiguous()
)
if betas is None:
betas = torch.zeros([batch_size, self.num_betas], dtype=dtype, device=device)
if transl is None:
transl = torch.zeros([batch_size, 3], dtype=dtype, device=device)
full_pose = torch.cat([global_orient, hand_pose], dim=1)
vertices, joints = lbs(
betas,
full_pose,
self.v_template,
self.shapedirs,
self.posedirs,
self.J_regressor,
self.parents,
self.lbs_weights,
pose2rot=False,
)
if self.joint_mapper is not None:
joints = self.joint_mapper(joints)
if transl is not None:
joints = joints + transl.unsqueeze(dim=1)
vertices = vertices + transl.unsqueeze(dim=1)
output = MANOOutput(
vertices=vertices if return_verts else None,
joints=joints if return_verts else None,
betas=betas,
global_orient=global_orient,
hand_pose=hand_pose,
full_pose=full_pose if return_full_pose else None,
)
return output
class FLAME(SMPL):
NUM_JOINTS = 5
SHAPE_SPACE_DIM = 300
EXPRESSION_SPACE_DIM = 100
NECK_IDX = 0
def __init__(
self,
model_path: str,
data_struct=None,
num_expression_coeffs=10,
create_expression: bool = True,
expression: Optional[Tensor] = None,
create_neck_pose: bool = True,
neck_pose: Optional[Tensor] = None,
create_jaw_pose: bool = True,
jaw_pose: Optional[Tensor] = None,
create_leye_pose: bool = True,
leye_pose: Optional[Tensor] = None,
create_reye_pose=True,
reye_pose: Optional[Tensor] = None,
use_face_contour=False,
batch_size: int = 1,
gender: str = "neutral",
dtype: torch.dtype = torch.float32,
ext="pkl",
**kwargs,
) -> None:
"""FLAME model constructor
Parameters
----------
model_path: str
The path to the folder or to the file where the model
parameters are stored
num_expression_coeffs: int, optional
Number of expression components to use
(default = 10).
create_expression: bool, optional
Flag for creating a member variable for the expression space
(default = True).
expression: torch.tensor, optional, Bx10
The default value for the expression member variable.
(default = None)
create_neck_pose: bool, optional
Flag for creating a member variable for the neck pose.
(default = False)
neck_pose: torch.tensor, optional, Bx3
The default value for the neck pose variable.
(default = None)
create_jaw_pose: bool, optional
Flag for creating a member variable for the jaw pose.
(default = False)
jaw_pose: torch.tensor, optional, Bx3
The default value for the jaw pose variable.
(default = None)
create_leye_pose: bool, optional
Flag for creating a member variable for the left eye pose.
(default = False)
leye_pose: torch.tensor, optional, Bx10
The default value for the left eye pose variable.
(default = None)
create_reye_pose: bool, optional
Flag for creating a member variable for the right eye pose.
(default = False)
reye_pose: torch.tensor, optional, Bx10
The default value for the right eye pose variable.
(default = None)
use_face_contour: bool, optional
Whether to compute the keypoints that form the facial contour
batch_size: int, optional
The batch size used for creating the member variables
gender: str, optional
Which gender to load
dtype: torch.dtype
The data type for the created variables
"""
model_fn = f"FLAME_{gender.upper()}.{ext}"
flame_path = os.path.join(model_path, model_fn)
assert osp.exists(flame_path), "Path {} does not exist!".format(flame_path)
if ext == "npz":
file_data = np.load(flame_path, allow_pickle=True)
elif ext == "pkl":
with open(flame_path, "rb") as smpl_file:
file_data = pickle.load(smpl_file, encoding="latin1")
else:
raise ValueError("Unknown extension: {}".format(ext))
data_struct = Struct(**file_data)
super(FLAME, self).__init__(
model_path=model_path,
data_struct=data_struct,
dtype=dtype,
batch_size=batch_size,
gender=gender,
ext=ext,
**kwargs,
)
self.use_face_contour = use_face_contour
self.vertex_joint_selector.extra_joints_idxs = to_tensor([], dtype=torch.long)
if create_neck_pose:
if neck_pose is None:
default_neck_pose = torch.zeros([batch_size, 3], dtype=dtype)
else:
default_neck_pose = torch.tensor(neck_pose, dtype=dtype)
neck_pose_param = nn.Parameter(default_neck_pose, requires_grad=True)
self.register_parameter("neck_pose", neck_pose_param)
if create_jaw_pose:
if jaw_pose is None:
default_jaw_pose = torch.zeros([batch_size, 3], dtype=dtype)
else:
default_jaw_pose = torch.tensor(jaw_pose, dtype=dtype)
jaw_pose_param = nn.Parameter(default_jaw_pose, requires_grad=True)
self.register_parameter("jaw_pose", jaw_pose_param)
if create_leye_pose:
if leye_pose is None:
default_leye_pose = torch.zeros([batch_size, 3], dtype=dtype)
else:
default_leye_pose = torch.tensor(leye_pose, dtype=dtype)
leye_pose_param = nn.Parameter(default_leye_pose, requires_grad=True)
self.register_parameter("leye_pose", leye_pose_param)
if create_reye_pose:
if reye_pose is None:
default_reye_pose = torch.zeros([batch_size, 3], dtype=dtype)
else:
default_reye_pose = torch.tensor(reye_pose, dtype=dtype)
reye_pose_param = nn.Parameter(default_reye_pose, requires_grad=True)
self.register_parameter("reye_pose", reye_pose_param)
shapedirs = data_struct.shapedirs
if len(shapedirs.shape) < 3:
shapedirs = shapedirs[:, :, None]
if shapedirs.shape[-1] < self.SHAPE_SPACE_DIM + self.EXPRESSION_SPACE_DIM:
# print(f'WARNING: You are using a {self.name()} model, with only'
# ' 10 shape and 10 expression coefficients.')
expr_start_idx = 10
expr_end_idx = 20
num_expression_coeffs = min(num_expression_coeffs, 10)
else:
expr_start_idx = self.SHAPE_SPACE_DIM
expr_end_idx = self.SHAPE_SPACE_DIM + num_expression_coeffs
num_expression_coeffs = min(num_expression_coeffs, self.EXPRESSION_SPACE_DIM)
self._num_expression_coeffs = num_expression_coeffs
expr_dirs = shapedirs[:, :, expr_start_idx:expr_end_idx]
self.register_buffer("expr_dirs", to_tensor(to_np(expr_dirs), dtype=dtype))
if create_expression:
if expression is None:
default_expression = torch.zeros([batch_size, self.num_expression_coeffs],
dtype=dtype)
else:
default_expression = torch.tensor(expression, dtype=dtype)
expression_param = nn.Parameter(default_expression, requires_grad=True)
self.register_parameter("expression", expression_param)
# The pickle file that contains the barycentric coordinates for
# regressing the landmarks
landmark_bcoord_filename = osp.join(model_path, "flame_static_embedding.pkl")
with open(landmark_bcoord_filename, "rb") as fp:
landmarks_data = pickle.load(fp, encoding="latin1")
lmk_faces_idx = landmarks_data["lmk_face_idx"].astype(np.int64)
self.register_buffer("lmk_faces_idx", torch.tensor(lmk_faces_idx, dtype=torch.long))
lmk_bary_coords = landmarks_data["lmk_b_coords"]
self.register_buffer("lmk_bary_coords", torch.tensor(lmk_bary_coords, dtype=dtype))
if self.use_face_contour:
face_contour_path = os.path.join(model_path, "flame_dynamic_embedding.npy")
contour_embeddings = np.load(face_contour_path, allow_pickle=True,
encoding="latin1")[()]
dynamic_lmk_faces_idx = np.array(contour_embeddings["lmk_face_idx"], dtype=np.int64)
dynamic_lmk_faces_idx = torch.tensor(dynamic_lmk_faces_idx, dtype=torch.long)
self.register_buffer("dynamic_lmk_faces_idx", dynamic_lmk_faces_idx)
dynamic_lmk_b_coords = torch.tensor(contour_embeddings["lmk_b_coords"], dtype=dtype)
self.register_buffer("dynamic_lmk_bary_coords", dynamic_lmk_b_coords)
neck_kin_chain = find_joint_kin_chain(self.NECK_IDX, self.parents)
self.register_buffer("neck_kin_chain", torch.tensor(neck_kin_chain, dtype=torch.long))
@property
def num_expression_coeffs(self):
return self._num_expression_coeffs
def name(self) -> str:
return "FLAME"
def extra_repr(self):
msg = [
super(FLAME, self).extra_repr(),
f"Number of Expression Coefficients: {self.num_expression_coeffs}",
f"Use face contour: {self.use_face_contour}",
]
return "\n".join(msg)
def forward(
self,
betas: Optional[Tensor] = None,
global_orient: Optional[Tensor] = None,
neck_pose: Optional[Tensor] = None,
transl: Optional[Tensor] = None,
expression: Optional[Tensor] = None,
jaw_pose: Optional[Tensor] = None,
leye_pose: Optional[Tensor] = None,
reye_pose: Optional[Tensor] = None,
return_verts: bool = True,
return_full_pose: bool = False,
pose2rot: bool = True,
**kwargs,
) -> FLAMEOutput:
"""
Forward pass for the SMPLX model
Parameters
----------
global_orient: torch.tensor, optional, shape Bx3
If given, ignore the member variable and use it as the global
rotation of the body. Useful if someone wishes to predicts this
with an external model. (default=None)
betas: torch.tensor, optional, shape Bx10
If given, ignore the member variable `betas` and use it
instead. For example, it can used if shape parameters
`betas` are predicted from some external model.
(default=None)
expression: torch.tensor, optional, shape Bx10
If given, ignore the member variable `expression` and use it
instead. For example, it can used if expression parameters
`expression` are predicted from some external model.
jaw_pose: torch.tensor, optional, shape Bx3
If given, ignore the member variable `jaw_pose` and
use this instead. It should either joint rotations in
axis-angle format.
jaw_pose: torch.tensor, optional, shape Bx3
If given, ignore the member variable `jaw_pose` and
use this instead. It should either joint rotations in
axis-angle format.
transl: torch.tensor, optional, shape Bx3
If given, ignore the member variable `transl` and use it
instead. For example, it can used if the translation
`transl` is predicted from some external model.
(default=None)
return_verts: bool, optional
Return the vertices. (default=True)
return_full_pose: bool, optional
Returns the full axis-angle pose vector (default=False)
Returns
-------
output: ModelOutput
A named tuple of type `ModelOutput`
"""
# If no shape and pose parameters are passed along, then use the
# ones from the module
global_orient = (global_orient if global_orient is not None else self.global_orient)
jaw_pose = jaw_pose if jaw_pose is not None else self.jaw_pose
neck_pose = neck_pose if neck_pose is not None else self.neck_pose
leye_pose = leye_pose if leye_pose is not None else self.leye_pose
reye_pose = reye_pose if reye_pose is not None else self.reye_pose
betas = betas if betas is not None else self.betas
expression = expression if expression is not None else self.expression
apply_trans = transl is not None or hasattr(self, "transl")
if transl is None:
if hasattr(self, "transl"):
transl = self.transl
full_pose = torch.cat([global_orient, neck_pose, jaw_pose, leye_pose, reye_pose], dim=1)
batch_size = max(betas.shape[0], global_orient.shape[0], jaw_pose.shape[0])
# Concatenate the shape and expression coefficients
scale = int(batch_size / betas.shape[0])
if scale > 1:
betas = betas.expand(scale, -1)
shape_components = torch.cat([betas, expression], dim=-1)
shapedirs = torch.cat([self.shapedirs, self.expr_dirs], dim=-1)
vertices, joints = lbs(
shape_components,
full_pose,
self.v_template,
shapedirs,
self.posedirs,
self.J_regressor,
self.parents,
self.lbs_weights,
pose2rot=pose2rot,
)
lmk_faces_idx = (self.lmk_faces_idx.unsqueeze(dim=0).expand(batch_size, -1).contiguous())
lmk_bary_coords = self.lmk_bary_coords.unsqueeze(dim=0).repeat(self.batch_size, 1, 1)
if self.use_face_contour:
lmk_idx_and_bcoords = find_dynamic_lmk_idx_and_bcoords(
vertices,
full_pose,
self.dynamic_lmk_faces_idx,
self.dynamic_lmk_bary_coords,
self.neck_kin_chain,
pose2rot=True,
)
dyn_lmk_faces_idx, dyn_lmk_bary_coords = lmk_idx_and_bcoords
lmk_faces_idx = torch.cat([lmk_faces_idx, dyn_lmk_faces_idx], 1)
lmk_bary_coords = torch.cat([
lmk_bary_coords.expand(batch_size, -1, -1), dyn_lmk_bary_coords
], 1)
landmarks = vertices2landmarks(vertices, self.faces_tensor, lmk_faces_idx, lmk_bary_coords)
# Add any extra joints that might be needed
joints = self.vertex_joint_selector(vertices, joints)
# Add the landmarks to the joints
joints = torch.cat([joints, landmarks], dim=1)
# Map the joints to the current dataset
if self.joint_mapper is not None:
joints = self.joint_mapper(joints=joints, vertices=vertices)
if apply_trans:
joints += transl.unsqueeze(dim=1)
vertices += transl.unsqueeze(dim=1)
output = FLAMEOutput(
vertices=vertices if return_verts else None,
joints=joints,
betas=betas,
expression=expression,
global_orient=global_orient,
neck_pose=neck_pose,
jaw_pose=jaw_pose,
full_pose=full_pose if return_full_pose else None,
)
return output
class FLAMELayer(FLAME):
def __init__(self, *args, **kwargs) -> None:
""" FLAME as a layer model constructor """
super(FLAMELayer, self).__init__(
create_betas=False,
create_expression=False,
create_global_orient=False,
create_neck_pose=False,
create_jaw_pose=False,
create_leye_pose=False,
create_reye_pose=False,
*args,
**kwargs,
)
def forward(
self,
betas: Optional[Tensor] = None,
global_orient: Optional[Tensor] = None,
neck_pose: Optional[Tensor] = None,
transl: Optional[Tensor] = None,
expression: Optional[Tensor] = None,
jaw_pose: Optional[Tensor] = None,
leye_pose: Optional[Tensor] = None,
reye_pose: Optional[Tensor] = None,
return_verts: bool = True,
return_full_pose: bool = False,
pose2rot: bool = True,
**kwargs,
) -> FLAMEOutput:
"""
Forward pass for the SMPLX model
Parameters
----------
global_orient: torch.tensor, optional, shape Bx3x3
Global rotation of the body. Useful if someone wishes to
predicts this with an external model. It is expected to be in
rotation matrix format. (default=None)
betas: torch.tensor, optional, shape BxN_b
Shape parameters. For example, it can used if shape parameters
`betas` are predicted from some external model.
(default=None)
expression: torch.tensor, optional, shape BxN_e
If given, ignore the member variable `expression` and use it
instead. For example, it can used if expression parameters
`expression` are predicted from some external model.
jaw_pose: torch.tensor, optional, shape Bx3x3
Jaw pose. It should either joint rotations in
rotation matrix format.
transl: torch.tensor, optional, shape Bx3
Translation vector of the body.
For example, it can used if the translation
`transl` is predicted from some external model.
(default=None)
return_verts: bool, optional
Return the vertices. (default=True)
return_full_pose: bool, optional
Returns the full axis-angle pose vector (default=False)
Returns
-------
output: ModelOutput
A named tuple of type `ModelOutput`
"""
device, dtype = self.shapedirs.device, self.shapedirs.dtype
if global_orient is None:
batch_size = 1
global_orient = (
torch.eye(3, device=device,
dtype=dtype).view(1, 1, 3, 3).expand(batch_size, -1, -1, -1).contiguous()
)
else:
batch_size = global_orient.shape[0]
if neck_pose is None:
neck_pose = (
torch.eye(3, device=device,
dtype=dtype).view(1, 1, 3, 3).expand(batch_size, 1, -1, -1).contiguous()
)
if jaw_pose is None:
jaw_pose = (
torch.eye(3, device=device,
dtype=dtype).view(1, 1, 3, 3).expand(batch_size, -1, -1, -1).contiguous()
)
if leye_pose is None:
leye_pose = (
torch.eye(3, device=device,
dtype=dtype).view(1, 1, 3, 3).expand(batch_size, -1, -1, -1).contiguous()
)
if reye_pose is None:
reye_pose = (
torch.eye(3, device=device,
dtype=dtype).view(1, 1, 3, 3).expand(batch_size, -1, -1, -1).contiguous()
)
if betas is None:
betas = torch.zeros([batch_size, self.num_betas], dtype=dtype, device=device)
if expression is None:
expression = torch.zeros([batch_size, self.num_expression_coeffs],
dtype=dtype,
device=device)
if transl is None:
transl = torch.zeros([batch_size, 3], dtype=dtype, device=device)
full_pose = torch.cat([global_orient, neck_pose, jaw_pose, leye_pose, reye_pose], dim=1)
shape_components = torch.cat([betas, expression], dim=-1)
shapedirs = torch.cat([self.shapedirs, self.expr_dirs], dim=-1)
vertices, joints = lbs(
shape_components,
full_pose,
self.v_template,
shapedirs,
self.posedirs,
self.J_regressor,
self.parents,
self.lbs_weights,
pose2rot=False,
)
lmk_faces_idx = (self.lmk_faces_idx.unsqueeze(dim=0).expand(batch_size, -1).contiguous())
lmk_bary_coords = self.lmk_bary_coords.unsqueeze(dim=0).repeat(self.batch_size, 1, 1)
if self.use_face_contour:
lmk_idx_and_bcoords = find_dynamic_lmk_idx_and_bcoords(
vertices,
full_pose,
self.dynamic_lmk_faces_idx,
self.dynamic_lmk_bary_coords,
self.neck_kin_chain,
pose2rot=False,
)
dyn_lmk_faces_idx, dyn_lmk_bary_coords = lmk_idx_and_bcoords
lmk_faces_idx = torch.cat([lmk_faces_idx, dyn_lmk_faces_idx], 1)
lmk_bary_coords = torch.cat([
lmk_bary_coords.expand(batch_size, -1, -1), dyn_lmk_bary_coords
], 1)
landmarks = vertices2landmarks(vertices, self.faces_tensor, lmk_faces_idx, lmk_bary_coords)
# Add any extra joints that might be needed
joints = self.vertex_joint_selector(vertices, joints)
# Add the landmarks to the joints
joints = torch.cat([joints, landmarks], dim=1)
# Map the joints to the current dataset
if self.joint_mapper is not None:
joints = self.joint_mapper(joints=joints, vertices=vertices)
joints += transl.unsqueeze(dim=1)
vertices += transl.unsqueeze(dim=1)
output = FLAMEOutput(
vertices=vertices if return_verts else None,
joints=joints,
betas=betas,
expression=expression,
global_orient=global_orient,
neck_pose=neck_pose,
jaw_pose=jaw_pose,
full_pose=full_pose if return_full_pose else None,
)
return output
def build_layer(model_path: str,
model_type: str = "smpl",
**kwargs) -> Union[SMPLLayer, SMPLHLayer, SMPLXLayer, MANOLayer, FLAMELayer]:
"""Method for creating a model from a path and a model type
Parameters
----------
model_path: str
Either the path to the model you wish to load or a folder,
where each subfolder contains the differents types, i.e.:
model_path:
|
|-- smpl
|-- SMPL_FEMALE
|-- SMPL_NEUTRAL
|-- SMPL_MALE
|-- smplh
|-- SMPLH_FEMALE
|-- SMPLH_MALE
|-- smplx
|-- SMPLX_FEMALE
|-- SMPLX_NEUTRAL
|-- SMPLX_MALE
|-- mano
|-- MANO RIGHT
|-- MANO LEFT
|-- flame
|-- FLAME_FEMALE
|-- FLAME_MALE
|-- FLAME_NEUTRAL
model_type: str, optional
When model_path is a folder, then this parameter specifies the
type of model to be loaded
**kwargs: dict
Keyword arguments
Returns
-------
body_model: nn.Module
The PyTorch module that implements the corresponding body model
Raises
------
ValueError: In case the model type is not one of SMPL, SMPLH,
SMPLX, MANO or FLAME
"""
if osp.isdir(model_path):
model_path = os.path.join(model_path, model_type)
else:
model_type = osp.basename(model_path).split("_")[0].lower()
if model_type.lower() == "smpl":
return SMPLLayer(model_path, **kwargs)
elif model_type.lower() == "smplh":
return SMPLHLayer(model_path, **kwargs)
elif model_type.lower() == "smplx":
return SMPLXLayer(model_path, **kwargs)
elif "mano" in model_type.lower():
return MANOLayer(model_path, **kwargs)
elif "flame" in model_type.lower():
return FLAMELayer(model_path, **kwargs)
else:
raise ValueError(f"Unknown model type {model_type}, exiting!")
def create(model_path: str,
model_type: str = "smpl",
**kwargs) -> Union[SMPL, SMPLH, SMPLX, MANO, FLAME]:
"""Method for creating a model from a path and a model type
Parameters
----------
model_path: str
Either the path to the model you wish to load or a folder,
where each subfolder contains the differents types, i.e.:
model_path:
|
|-- smpl
|-- SMPL_FEMALE
|-- SMPL_NEUTRAL
|-- SMPL_MALE
|-- smplh
|-- SMPLH_FEMALE
|-- SMPLH_MALE
|-- smplx
|-- SMPLX_FEMALE
|-- SMPLX_NEUTRAL
|-- SMPLX_MALE
|-- mano
|-- MANO RIGHT
|-- MANO LEFT
model_type: str, optional
When model_path is a folder, then this parameter specifies the
type of model to be loaded
**kwargs: dict
Keyword arguments
Returns
-------
body_model: nn.Module
The PyTorch module that implements the corresponding body model
Raises
------
ValueError: In case the model type is not one of SMPL, SMPLH,
SMPLX, MANO or FLAME
"""
# If it's a folder, assume
if osp.isdir(model_path):
model_path = os.path.join(model_path, model_type)
else:
model_type = osp.basename(model_path).split("_")[0].lower()
if model_type.lower() == "smpl":
return SMPL(model_path, **kwargs)
elif model_type.lower() == "smplh":
return SMPLH(model_path, **kwargs)
elif model_type.lower() == "smplx":
return SMPLX(model_path, **kwargs)
elif "mano" in model_type.lower():
return MANO(model_path, **kwargs)
elif "flame" in model_type.lower():
return FLAME(model_path, **kwargs)
else:
raise ValueError(f"Unknown model type {model_type}, exiting!")