handle smplh

Author: CalciferZh

.gitignore

@@ -1,4 +1,5 @@
 model_files/
+.vscode/
 *.obj
 
 # Byte-compiled / optimized / DLL files

armatures.py

@@ -3,10 +3,10 @@
 class MANOArmature:
   n_joints = 16
 
-  # indices of extended joints (finger tips)
-  joints_ext = [333, 444, 672, 555, 744]
+  # indices of extended keypoints
+  keypoints_ext = [333, 444, 672, 555, 744]
 
-  n_keypoints = n_joints + len(joints_ext)
+  n_keypoints = n_joints + len(keypoints_ext)
 
   root = 0
 
@@ -19,18 +19,19 @@ class MANOArmature:
     'L0', 'L1', 'L2', #9
     'R0', 'R1', 'R2', #12
     'T0', 'T1', 'T2', #15
-    'I3', 'M3', 'L3', 'R3', 'T3' #20, tips are manually added (not in MANO)
+    # extended
+    'I3', 'M3', 'L3', 'R3', 'T3' #20
   ]
 
 
 class SMPLArmature:
   n_joints = 24
 
-  # indices of extended joints (limb ends)
+  # indices of extended keypoints (limb ends)
   # lfinger, rfinger, ltoe, rtoe, head-top
-  joints_ext = [2446, 5907, 3216, 6618, 411]
+  keypoints_ext = [2446, 5907, 3216, 6618, 411]
 
-  n_keypoints = n_joints + len(joints_ext)
+  n_keypoints = n_joints + len(keypoints_ext)
 
   labels = [
     'pelvis',
@@ -48,4 +49,41 @@ class SMPLArmature:
     'lelbow', 'relbow',
     'lwrist', 'rwrist',
     'lhand', 'rhand'
+    # extended
+    'lfinger_tip', 'rfinger_tip', 'ltoe_tip', 'r_toe_tip', 'head_top'
+  ]
+
+
+class SMPLHArmature:
+  n_joints = 52
+
+  # indices of extended keypoints (limb ends)
+  keypoints_ext = [
+    2746, 2320, 2446, 2557, 2674,
+    6191, 5781, 5907, 6018, 6135,
+    3216, 6618, 411
+  ]
+
+  n_keypoints = n_joints + len(keypoints_ext)
+
+  labels = [
+    'pelvis',
+    'llegroot', 'rlegroot',
+    'lowerback',
+    'lknee', 'rknee',
+    'upperback',
+    'lankle', 'rankle',
+    'thorax',
+    'ltoes', 'rtoes',
+    'lowerneck',
+    'lclavicle', 'rclavicle',
+    'upperneck',
+    'larmroot', 'rarmroot',
+    'lelbow', 'relbow',
+    'lwrist', 'rwrist',
+    'lhand', 'rhand'
+    # extended
+    'left-thumb', 'li', 'lm', 'lr', 'll',
+    'rt', 'ri', 'rm', 'rr', 'rl',
+    'ltoe-tip', 'rtoe-tip', 'heat-top'
   ]

config.py

@@ -1,5 +1,7 @@
 MANO_MODEL_PATH = './model_files/mano_left.pkl'
 SMPL_MODEL_PATH = './model_files/smpl_female.pkl'
+SMPLH_MODEL_PATH = './model_files/smplh_neutral'
 
 OFFICIAL_MANO_PATH = './model_files/mano_v1_2/mano_v1_2/models/MANO_LEFT.pkl'
 OFFICIAL_SMPL_PATH = './model_files/SMPL_python_v.1.0.0/smpl/models/basicModel_f_lbs_10_207_0_v1.0.0.pkl'
+OFFICIAL_SMPLH_PATH = './model_files/smplh/neutral/model.npz'

example.py

@@ -4,25 +4,33 @@
 import numpy as np
 import config
 
-n_shape = 10
+
 np.random.seed(20160923)
+pose_glb = np.zeros([1, 3])
+
 
 ########################## mano settings #########################
 # n_pose = 12
 # n_shape = 10
-# np.random.seed(20160923)
 # pose_pca = np.random.normal(size=n_pose)
-# pose_glb = np.zeros([1, 3])
 # shape = np.random.normal(size=n_shape)
 # mesh = KinematicModel(config.MANO_MODEL_PATH, MANOArmature, scale=1000)
 
 
 ########################## smpl settings ##########################
-n_pose = 23 * 3
+# n_pose = 23 * 3
+# n_shape = 10
+# pose_pca = np.random.uniform(-0.2, 0.2, size=n_pose)
+# shape = np.random.normal(size=n_shape)
+# mesh = KinematicModel(config.SMPL_MODEL_PATH, SMPLArmature, scale=10)
+
+
+########################## smpl-h settings ##########################
+n_pose = 51 * 3
+n_shape = 16
 pose_pca = np.random.uniform(-0.2, 0.2, size=n_pose)
-pose_glb = np.zeros([1, 3])
 shape = np.random.normal(size=n_shape)
-mesh = KinematicModel(config.SMPL_MODEL_PATH, SMPLArmature, scale=10)
+mesh = KinematicModel(config.SMPLH_MODEL_PATH, SMPLHArmature, scale=10)
 
 
 ########################## solving example ############################
@@ -36,13 +44,13 @@
 
 shape_est, pose_pca_est, pose_glb_est = wrapper.decode(params_est)
 
-print('-----------------------------------------------------------------------')
+print('----------------------------------------------------------------------')
 print('ground truth parameters')
 print('pose pca coefficients:', pose_pca)
 print('pose global rotation:', pose_glb)
 print('shape: pca coefficients:', shape)
 
-print('-----------------------------------------------------------------------')
+print('----------------------------------------------------------------------')
 print('estimated parameters')
 print('pose pca coefficients:', pose_pca_est)
 print('pose global rotation:', pose_glb_est)

models.py

@@ -22,7 +22,7 @@ def __init__(self, model_path, armature, scale=1):
 
       self.parents = params['parents']
 
-    self.n_shape_params = 10
+    self.n_shape_params = self.mesh_shape_basis.shape[-1]
     self.scale = scale
 
     self.armature = armature
@@ -37,7 +37,7 @@ def __init__(self, model_path, armature, scale=1):
     self.J_regressor_ext = \
       np.zeros([self.armature.n_keypoints, self.J_regressor.shape[1]])
     self.J_regressor_ext[:self.armature.n_joints] = self.J_regressor
-    for i, v in enumerate(self.armature.joints_ext):
+    for i, v in enumerate(self.armature.keypoints_ext):
       self.J_regressor_ext[i + self.armature.n_joints, v] = 1
 
     self.update()
@@ -170,7 +170,7 @@ class KinematicPCAWrapper():
   def __init__(self, core, n_pose=12):
     self.core = core
     self.n_pose = n_pose
-    self.n_shape = 10
+    self.n_shape = core.n_shape_params
     self.n_glb = 3
     self.n_params = self.n_pose + self.n_shape + self.n_glb
 

prepare_model.py

@@ -44,5 +44,25 @@ def prepare_smpl_model():
     pickle.dump(params, f)
 
 
+def prepare_smplh_model():
+  data = np.load(OFFICIAL_SMPLH_PATH)
+  params = {
+    # SMPL does not provide pose PCA
+    'pose_pca_basis': np.eye(51 * 3),
+    'pose_pca_mean': np.zeros(51 * 3),
+    'J_regressor': data['J_regressor'],
+    'skinning_weights': np.array(data['weights']),
+    # pose blend shape
+    'mesh_pose_basis': np.array(data['posedirs']),
+    'mesh_shape_basis': np.array(data['shapedirs']),
+    'mesh_template': np.array(data['v_template']),
+    'faces': np.array(data['f']),
+    'parents': data['kintree_table'][0].tolist(),
+  }
+  params['parents'][0] = None
+  with open(SMPLH_MODEL_PATH, 'wb') as f:
+    pickle.dump(params, f)
+
+
 if __name__ == '__main__':
-  prepare_smpl_model()
+  prepare_smplh_model()