release candidate

Author: CalciferZh

README.md

@@ -1 +1,25 @@
-# Minimal-IK
+# Minimal-IK
+
+A simple and naive inverse kinematics solver for MANO hand model, SMPL body model, and SMPLH body+hand model.
+
+Briefly, given joint coordinates (and optional other keypoints), the solver gives the corresponding model parameters.
+
+Levenberg–Marquardt algorithm is used, the energy is simply the L2 distance between the keypoints.
+
+## Usage
+
+### Models
+
+1. Download the official model from MPI.
+2. See `config.py` and set the official model path.
+3. See `prepare_model.py`, use the provided function to pre-process the model.
+
+### Solver
+
+1. See `example.py`, un-comment the corresponding code.
+2. `python example.py`.
+3. The example ground truth mesh and estimated mesh are saved to `gt.obj` and `est.obj` respectively.
+
+### Dependencies
+
+Every required package is available via `pip install`.

armatures.py

@@ -1,17 +1,14 @@
 # it must be assured that parent joint appears before child joint
 
 class MANOArmature:
+  # number of movable joints
   n_joints = 16
 
   # indices of extended keypoints
   keypoints_ext = [333, 444, 672, 555, 744]
 
   n_keypoints = n_joints + len(keypoints_ext)
 
-  root = 0
-
-  center = 4
-
   labels = [
     'W', #0
     'I0', 'I1', 'I2', #3
@@ -27,37 +24,35 @@ class MANOArmature:
 class SMPLArmature:
   n_joints = 24
 
-  # indices of extended keypoints (limb ends)
-  # lfinger, rfinger, ltoe, rtoe, head-top
   keypoints_ext = [2446, 5907, 3216, 6618, 411]
 
   n_keypoints = n_joints + len(keypoints_ext)
 
   labels = [
     'pelvis',
-    'llegroot', 'rlegroot',
+    'left leg root', 'right leg root',
     'lowerback',
-    'lknee', 'rknee',
+    'left knee', 'right knee',
     'upperback',
-    'lankle', 'rankle',
+    'left ankle', 'right ankle',
     'thorax',
-    'ltoes', 'rtoes',
+    'left toes', 'right toes',
     'lowerneck',
-    'lclavicle', 'rclavicle',
+    'left clavicle', 'right clavicle',
     'upperneck',
-    'larmroot', 'rarmroot',
-    'lelbow', 'relbow',
-    'lwrist', 'rwrist',
-    'lhand', 'rhand'
+    'left armroot', 'right armroot',
+    'left elbow', 'right elbow',
+    'left wrist', 'right wrist',
+    'left hand', 'right hand'
     # extended
-    'lfinger_tip', 'rfinger_tip', 'ltoe_tip', 'r_toe_tip', 'head_top'
+    'left finger tip', 'right finger tip', 'left toe tip', 'right toe tip',
+    'head_top'
   ]
 
 
 class SMPLHArmature:
   n_joints = 52
 
-  # indices of extended keypoints (limb ends)
   keypoints_ext = [
     2746, 2320, 2446, 2557, 2674,
     6191, 5781, 5907, 6018, 6135,
@@ -68,22 +63,22 @@ class SMPLHArmature:
 
   labels = [
     'pelvis',
-    'llegroot', 'rlegroot',
+    'left leg root', 'right leg root',
     'lowerback',
-    'lknee', 'rknee',
+    'left knee', 'right knee',
     'upperback',
-    'lankle', 'rankle',
+    'left ankle', 'right ankle',
     'thorax',
-    'ltoes', 'rtoes',
+    'left toes', 'right toes',
     'lowerneck',
-    'lclavicle', 'rclavicle',
+    'left clavicle', 'right clavicle',
     'upperneck',
-    'larmroot', 'rarmroot',
-    'lelbow', 'relbow',
-    'lwrist', 'rwrist',
-    'lhand', 'rhand'
+    'left armroot', 'right armroot',
+    'left elbow', 'right elbow',
+    'left wrist', 'right wrist',
+    'left hand', 'right hand'
     # extended
-    'left-thumb', 'li', 'lm', 'lr', 'll',
-    'rt', 'ri', 'rm', 'rr', 'rl',
-    'ltoe-tip', 'rtoe-tip', 'heat-top'
+    'left thumb', 'left index', 'left middle', 'left ring', 'left little',
+    'right thumb', 'right index', 'right middle', 'right ring', 'right little',
+    'left toe tip', 'right toe tip', 'heat-top'
   ]

example.py

@@ -6,31 +6,35 @@
 
 
 np.random.seed(20160923)
-pose_glb = np.zeros([1, 3])
+pose_glb = np.zeros([1, 3]) # global rotation
 
 
 ########################## mano settings #########################
-# n_pose = 12
-# n_shape = 10
-# pose_pca = np.random.normal(size=n_pose)
-# shape = np.random.normal(size=n_shape)
-# mesh = KinematicModel(config.MANO_MODEL_PATH, MANOArmature, scale=1000)
+n_pose = 12 # number of pose pca coefficients, in mano the maximum is 45
+n_shape = 10 # number of shape pca coefficients
+pose_pca = np.random.normal(size=n_pose)
+shape = np.random.normal(size=n_shape)
+mesh = KinematicModel(config.MANO_MODEL_PATH, MANOArmature, scale=1000)
 
 
 ########################## smpl settings ##########################
-# n_pose = 23 * 3
+# note that in smpl and smpl-h no pca for pose is provided
+# therefore in the model we fake an identity matrix as the pca coefficients
+# to make the code compatible
+
+# n_pose = 23 * 3 # degrees of freedom, (n_joints - 1) * 3
 # n_shape = 10
 # pose_pca = np.random.uniform(-0.2, 0.2, size=n_pose)
 # shape = np.random.normal(size=n_shape)
 # mesh = KinematicModel(config.SMPL_MODEL_PATH, SMPLArmature, scale=10)
 
 
 ########################## smpl-h settings ##########################
-n_pose = 51 * 3
-n_shape = 16
-pose_pca = np.random.uniform(-0.2, 0.2, size=n_pose)
-shape = np.random.normal(size=n_shape)
-mesh = KinematicModel(config.SMPLH_MODEL_PATH, SMPLHArmature, scale=10)
+# n_pose = 51 * 3
+# n_shape = 16
+# pose_pca = np.random.uniform(-0.2, 0.2, size=n_pose)
+# shape = np.random.normal(size=n_shape)
+# mesh = KinematicModel(config.SMPLH_MODEL_PATH, SMPLHArmature, scale=10)
 
 
 ########################## solving example ############################

models.py

@@ -3,7 +3,21 @@
 
 
 class KinematicModel():
+  """
+  Kinematic model that takes in model parameters and outputs mesh, keypoints,
+  etc.
+  """
   def __init__(self, model_path, armature, scale=1):
+    """
+    Parameters
+    ----------
+    model_path : str
+      Path to the model to be loaded.
+    armature : object
+      An armature class from `aramatures.py`.
+    scale : int, optional
+      Scale of the model to make the solving easier, by default 1
+    """
     with open(model_path, 'rb') as f:
       params = pickle.load(f)
 
@@ -14,7 +28,7 @@ def __init__(self, model_path, armature, scale=1):
 
       self.skinning_weights = params['skinning_weights']
 
-      self.mesh_pose_basis = params['mesh_pose_basis']
+      self.mesh_pose_basis = params['mesh_pose_basis'] # pose blend shape
       self.mesh_shape_basis = params['mesh_shape_basis']
       self.mesh_template = params['mesh_template']
 
@@ -43,6 +57,28 @@ def __init__(self, model_path, armature, scale=1):
     self.update()
 
   def set_params(self, pose_abs=None, pose_pca=None, pose_glb=None, shape=None):
+    """
+    Set model parameters and get the mesh. Do not set `pose_abs` and `pose_pca`
+    at the same time.
+
+    Parameters
+    ----------
+    pose_abs : np.ndarray, shape [n_joints, 3], optional
+      The absolute model pose in axis-angle, by default None
+    pose_pca : np.ndarray, optional
+      The PCA coefficients of the pose, shape [n_pose, 3], by default None
+    pose_glb : np.ndarray, shape [1, 3], optional
+      Global rotation for the model, by default None
+    shape : np.ndarray, shape [n_shape], optional
+      Shape coefficients of the pose, by default None
+
+    Returns
+    -------
+    np.ndarray, shape [N, 3]
+      Vertices coordinates of the mesh, scale applied.
+    np.ndarray, shape [K, 3]
+      Keypoints coordinates of the model, scale applied.
+    """
     if pose_abs is not None:
       self.pose = pose_abs
     elif pose_pca is not None:
@@ -59,6 +95,16 @@ def set_params(self, pose_abs=None, pose_pca=None, pose_glb=None, shape=None):
     return self.update()
 
   def update(self):
+    """
+    Re-compute vertices and keypoints with given parameters.
+
+    Returns
+    -------
+    np.ndarray, shape [N, 3]
+      Vertices coordinates of the mesh, scale applied.
+    np.ndarray, shape [K, 3]
+      Keypoints coordinates of the model, scale applied.
+    """
     verts = self.mesh_template + self.mesh_shape_basis.dot(self.shape)
     self.J = self.J_regressor.dot(verts)
     self.R = self.rodrigues(self.pose.reshape((-1, 1, 3)))
@@ -167,19 +213,60 @@ def save_obj(self, path):
 
 
 class KinematicPCAWrapper():
+  """
+  A wrapper for `KinematicsModel` to be compatible to the solver.
+  """
   def __init__(self, core, n_pose=12):
+    """
+    Parameters
+    ----------
+    core : KinematicModel
+      Core model to be manipulated.
+    n_pose : int, optional
+      Degrees of freedom for pose, by default 12
+    """
     self.core = core
     self.n_pose = n_pose
     self.n_shape = core.n_shape_params
     self.n_glb = 3
     self.n_params = self.n_pose + self.n_shape + self.n_glb
 
   def run(self, params):
+    """
+    Set the parameters, return the corresponding result.
+
+    Parameters
+    ----------
+    params : np.ndarray
+      Model parameters.
+
+    Returns
+    -------
+    np.ndarray
+      Corresponding result.
+    """
     shape, pose_pca, pose_glb = self.decode(params)
     return \
       self.core.set_params(pose_glb=pose_glb, pose_pca=pose_pca, shape=shape)[1]
 
   def decode(self, params):
+    """
+    Decode the compact model parameters into semantic parameters.
+
+    Parameters
+    ----------
+    params : np.ndarray
+      Model parameters.
+
+    Returns
+    -------
+    np.ndarray
+      Shape parameters.
+    np.ndarray
+      Pose parameters.
+    np.ndarray
+      Global rotation.
+    """
     pose_glb = params[:self.n_glb]
     pose_pca = params[self.n_glb:-self.n_shape]
     shape = params[-self.n_shape:]

prepare_model.py

@@ -4,6 +4,9 @@
 
 
 def prepare_mano_model():
+  """
+  Convert the official MANO model into compatible format with this project.
+  """
   with open(OFFICIAL_MANO_PATH, 'rb') as f:
     data = pickle.load(f, encoding='latin1')
   params = {
@@ -24,6 +27,9 @@ def prepare_mano_model():
 
 
 def prepare_smpl_model():
+  """
+  Convert the official SMPL model into compatible format with this project.
+  """
   with open(OFFICIAL_SMPL_PATH, 'rb') as f:
     data = pickle.load(f, encoding='latin1')
   params = {
@@ -45,6 +51,9 @@ def prepare_smpl_model():
 
 
 def prepare_smplh_model():
+  """
+  Convert the official SMPLH model into compatible format with this project.
+  """
   data = np.load(OFFICIAL_SMPLH_PATH)
   params = {
     # SMPL does not provide pose PCA