These changes further improve point normalization to make it easier to detect expressions under rotations.
This commit is contained in:
Emiliana
parent
9c12d29399
commit
31536929f7
2 changed files with 72 additions and 37 deletions
|
|
@ -94,6 +94,8 @@ public class OpenSeeExpression : MonoBehaviour
|
|||
public bool pointsLipLower = true;
|
||||
[Tooltip("When enabled, certain manually designed features will be used to determine the expression.")]
|
||||
public bool features = true;
|
||||
[Tooltip("When enabled, the depth value of points is also used for training.")]
|
||||
public bool includeDepth = false;
|
||||
}
|
||||
[Serializable]
|
||||
private class OpenSeeExpressionRepresentation {
|
||||
|
|
@ -171,7 +173,7 @@ public class OpenSeeExpression : MonoBehaviour
|
|||
private int[] indicesBrowLeft = new int[] {22, 23, 24, 25, 26};
|
||||
private int[] indicesEyeRight = new int[] {36, 37, 38, 39, 40, 41};
|
||||
private int[] indicesEyeLeft = new int[] {42, 43, 44, 45, 46, 47};
|
||||
private int[] indicesNose = new int[] {27, 28, 29, 30, 31, 32, 33, 34, 35};
|
||||
private int[] indicesNose = new int[] {27, 28, 29, /*30, */31, 32, 33, 34, 35};
|
||||
private int[] indicesMouthCorner = new int[] {58, 62};
|
||||
private int[] indicesLipUpper = new int[] {48, 49, 50, 51, 52, 59, 60, 61};
|
||||
private int[] indicesLipLower = new int[] {53, 54, 55, 56, 57, 63, 64, 65};
|
||||
|
|
@ -186,64 +188,91 @@ public class OpenSeeExpression : MonoBehaviour
|
|||
foreach (int i in indicesFaceContour) {
|
||||
indexList.Add(colsBase + i * 3);
|
||||
indexList.Add(colsBase + i * 3 + 1);
|
||||
indexList.Add(colsBase + i * 3 + 2);
|
||||
cols += 3;
|
||||
cols += 2;
|
||||
if (pointSelection.includeDepth) {
|
||||
indexList.Add(colsBase + i * 3 + 2);
|
||||
cols++;
|
||||
}
|
||||
}
|
||||
if (pointSelection.pointsBrowRight)
|
||||
foreach (int i in indicesBrowRight) {
|
||||
indexList.Add(colsBase + i * 3);
|
||||
indexList.Add(colsBase + i * 3 + 1);
|
||||
indexList.Add(colsBase + i * 3 + 2);
|
||||
cols += 3;
|
||||
cols += 2;
|
||||
if (pointSelection.includeDepth) {
|
||||
indexList.Add(colsBase + i * 3 + 2);
|
||||
cols++;
|
||||
}
|
||||
}
|
||||
if (pointSelection.pointsBrowLeft)
|
||||
foreach (int i in indicesBrowLeft) {
|
||||
indexList.Add(colsBase + i * 3);
|
||||
indexList.Add(colsBase + i * 3 + 1);
|
||||
indexList.Add(colsBase + i * 3 + 2);
|
||||
cols += 3;
|
||||
cols += 2;
|
||||
if (pointSelection.includeDepth) {
|
||||
indexList.Add(colsBase + i * 3 + 2);
|
||||
cols++;
|
||||
}
|
||||
}
|
||||
if (pointSelection.pointsEyeRight)
|
||||
foreach (int i in indicesEyeRight) {
|
||||
indexList.Add(colsBase + i * 3);
|
||||
indexList.Add(colsBase + i * 3 + 1);
|
||||
indexList.Add(colsBase + i * 3 + 2);
|
||||
cols += 3;
|
||||
cols += 2;
|
||||
if (pointSelection.includeDepth) {
|
||||
indexList.Add(colsBase + i * 3 + 2);
|
||||
cols++;
|
||||
}
|
||||
}
|
||||
if (pointSelection.pointsEyeLeft)
|
||||
foreach (int i in indicesEyeLeft) {
|
||||
indexList.Add(colsBase + i * 3);
|
||||
indexList.Add(colsBase + i * 3 + 1);
|
||||
indexList.Add(colsBase + i * 3 + 2);
|
||||
cols += 3;
|
||||
cols += 2;
|
||||
if (pointSelection.includeDepth) {
|
||||
indexList.Add(colsBase + i * 3 + 2);
|
||||
cols++;
|
||||
}
|
||||
}
|
||||
if (pointSelection.pointsNose)
|
||||
foreach (int i in indicesNose) {
|
||||
indexList.Add(colsBase + i * 3);
|
||||
indexList.Add(colsBase + i * 3 + 1);
|
||||
indexList.Add(colsBase + i * 3 + 2);
|
||||
cols += 3;
|
||||
cols += 2;
|
||||
if (pointSelection.includeDepth) {
|
||||
indexList.Add(colsBase + i * 3 + 2);
|
||||
cols++;
|
||||
}
|
||||
}
|
||||
if (pointSelection.pointsNose)
|
||||
foreach (int i in indicesMouthCorner) {
|
||||
indexList.Add(colsBase + i * 3);
|
||||
indexList.Add(colsBase + i * 3 + 1);
|
||||
indexList.Add(colsBase + i * 3 + 2);
|
||||
cols += 3;
|
||||
cols += 2;
|
||||
if (pointSelection.includeDepth) {
|
||||
indexList.Add(colsBase + i * 3 + 2);
|
||||
cols++;
|
||||
}
|
||||
}
|
||||
if (pointSelection.pointsLipUpper)
|
||||
foreach (int i in indicesLipUpper) {
|
||||
indexList.Add(colsBase + i * 3);
|
||||
indexList.Add(colsBase + i * 3 + 1);
|
||||
indexList.Add(colsBase + i * 3 + 2);
|
||||
cols += 3;
|
||||
cols += 2;
|
||||
if (pointSelection.includeDepth) {
|
||||
indexList.Add(colsBase + i * 3 + 2);
|
||||
cols++;
|
||||
}
|
||||
}
|
||||
if (pointSelection.pointsLipLower)
|
||||
foreach (int i in indicesLipLower) {
|
||||
indexList.Add(colsBase + i * 3);
|
||||
indexList.Add(colsBase + i * 3 + 1);
|
||||
indexList.Add(colsBase + i * 3 + 2);
|
||||
cols += 3;
|
||||
cols += 2;
|
||||
if (pointSelection.includeDepth) {
|
||||
indexList.Add(colsBase + i * 3 + 2);
|
||||
cols++;
|
||||
}
|
||||
}
|
||||
if (pointSelection.features)
|
||||
for (int i = 0; i < 14; i++) {
|
||||
|
|
|
|||
42
tracker.py
42
tracker.py
|
|
@ -307,22 +307,27 @@ class FaceInfo():
|
|||
def update_contour(self):
|
||||
self.contour = np.array(self.face_3d[self.contour_pts])
|
||||
|
||||
def normalize_pts3d(self, pts_3d):
|
||||
# Calculate angle using nose
|
||||
pts_3d[:, 0:2] -= pts_3d[30, 0:2]
|
||||
alpha = angle(pts_3d[30, 0:2], pts_3d[27, 0:2])
|
||||
alpha -= np.deg2rad(90)
|
||||
|
||||
R = np.matrix([[np.cos(alpha), -np.sin(alpha)], [np.sin(alpha), np.cos(alpha)]])
|
||||
pts_3d[:, 0:2] = (pts_3d - pts_3d[30])[:, 0:2].dot(R) + pts_3d[30, 0:2]
|
||||
|
||||
# Vertical scale
|
||||
pts_3d[:, 1] /= np.mean((pts_3d[27:30, 1] - pts_3d[28:31, 1]) / (self.tracker.face_3d[27:30, 1] - self.tracker.face_3d[28:31, 1]))
|
||||
|
||||
# Horizontal scale
|
||||
pts_3d[:, 0] /= np.mean(np.abs(pts_3d[[0, 36, 42], 0] - pts_3d[[16, 39, 45], 0]) / np.abs(self.tracker.face_3d[[0, 36, 42], 0] - self.tracker.face_3d[[16, 39, 45], 0]))
|
||||
|
||||
return pts_3d
|
||||
|
||||
def adjust_3d(self):
|
||||
if self.conf < 0.4 or self.pnp_error > 300:
|
||||
return
|
||||
|
||||
norm = np.array([
|
||||
((self.pts_3d[0, 0] - self.pts_3d[16, 0]) + (self.pts_3d[1, 0] - self.pts_3d[15, 0])) / 2.0,
|
||||
((self.pts_3d[27, 1] - self.pts_3d[28, 1]) + (self.pts_3d[28, 1] - self.pts_3d[29, 1])) / 2.0,
|
||||
np.max(self.pts_3d[0:66, 2]) - np.min(self.pts_3d[0:66, 2])
|
||||
])
|
||||
self.pts_3d = (self.pts_3d / norm) * self.tracker.norm
|
||||
|
||||
self.current_features = self.features.update(self.pts_3d[:, 0:2])
|
||||
self.eye_blink = []
|
||||
self.eye_blink.append(1 - min(max(0, -self.current_features["eye_r"]), 1))
|
||||
self.eye_blink.append(1 - min(max(0, -self.current_features["eye_l"]), 1))
|
||||
|
||||
max_runs = 1
|
||||
eligible = np.arange(0, 66)
|
||||
changed_any = False
|
||||
|
|
@ -392,6 +397,13 @@ class FaceInfo():
|
|||
self.face_3d[update_indices] = self.face_3d[update_indices] * weights[update_indices] + updated[update_indices] * (1. - weights[update_indices])
|
||||
self.update_contour()
|
||||
|
||||
self.pts_3d = self.normalize_pts3d(self.pts_3d)
|
||||
self.current_features = self.features.update(self.pts_3d[:, 0:2])
|
||||
self.eye_blink = []
|
||||
self.eye_blink.append(1 - min(max(0, -self.current_features["eye_r"]), 1))
|
||||
self.eye_blink.append(1 - min(max(0, -self.current_features["eye_l"]), 1))
|
||||
|
||||
|
||||
class Tracker():
|
||||
def __init__(self, width, height, model_type=3, threshold=0.4, max_faces=1, discard_after=5, scan_every=3, bbox_growth=0.0, max_threads=4, silent=False, model_dir=None, no_gaze=False, use_retinaface=False):
|
||||
options = onnxruntime.SessionOptions()
|
||||
|
|
@ -532,12 +544,6 @@ class Tracker():
|
|||
[-0.25799, 0.27608, -0.24967],
|
||||
], np.float32) * np.array([1.0, 1.0, 1.3])
|
||||
|
||||
self.norm = np.array([
|
||||
((self.face_3d[0, 0] - self.face_3d[16, 0]) + (self.face_3d[1, 0] - self.face_3d[15, 0])) / 2.0,
|
||||
((self.face_3d[27, 1] - self.face_3d[28, 1]) + (self.face_3d[28, 1] - self.face_3d[29, 1])) / 2.0,
|
||||
np.max(self.face_3d[0:66, 2]) - np.min(self.face_3d[0:66, 2])
|
||||
])
|
||||
|
||||
self.camera = np.array([[width, 0, width/2], [0, width, height/2], [0, 0, 1]], np.float32)
|
||||
self.inverse_camera = np.linalg.inv(self.camera)
|
||||
self.dist_coeffs = np.zeros((4,1))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue