From 6b876be62388651ff8c727086e36445b39f1e809 Mon Sep 17 00:00:00 2001 From: Emiliana <38952746+emilianavt@users.noreply.github.com> Date: Sun, 9 Aug 2020 17:32:29 +0200 Subject: [PATCH 1/2] Optimized landmarks function for a 1-2ms speedup. --- tracker.py | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/tracker.py b/tracker.py index 91f0967..a9663c7 100644 --- a/tracker.py +++ b/tracker.py @@ -110,6 +110,10 @@ def logit(p, factor=16.0): p = p/(1-p) return float(np.log(p)) / float(factor) +def logit_arr(p, factor=16.0): + p = np.clip(p, 0.0000001, 0.9999999) + return np.log(p / (1 - p)) / float(factor) + def matrix_to_quaternion(m): t = 0.0 q = [0.0, 0.0, 0, 0.0] @@ -613,8 +617,19 @@ class Tracker(): self.scan_every = scan_every self.bbox_growth = bbox_growth self.silent = silent + self.res = 224. + if model_type < 0: + self.res = 56. self.res_i = int(self.res) + self.out_res = 27. + if model_type < 0: + self.out_res = 13. + self.out_res_i = int(self.out_res) + 1 + self.logit_factor = 16. + if model_type < 0: + self.logit_factor = 8. + self.no_gaze = no_gaze self.debug_gaze = False self.max_feature_updates = max_feature_updates @@ -651,22 +666,19 @@ class Tracker(): def landmarks(self, tensor, crop_info): crop_x1, crop_y1, scale_x, scale_y, _ = crop_info avg_conf = 0 - lms = [] res = self.res - 1 - for i in range(0, 66): - m = int(tensor[i].argmax()) - x = m // 28 - y = m % 28 - conf = float(tensor[i][x,y]) - avg_conf = avg_conf + conf - off_x = res * ((1. * logit(tensor[66 + i][x, y])) - 0.0) - off_y = res * ((1. * logit(tensor[66 * 2 + i][x, y])) - 0.0) - off_x = math.floor(off_x + 0.5) - off_y = math.floor(off_y + 0.5) - lm_x = crop_y1 + scale_y * (res * (float(x) / 27.) + off_x) - lm_y = crop_x1 + scale_x * (res * (float(y) / 27.) + off_y) - lms.append((lm_x,lm_y,conf)) - avg_conf = avg_conf / 66. + t_main = tensor[0:66].reshape((66,self.out_res_i * self.out_res_i)) + t_m = t_main.argmax(1) + indices = np.expand_dims(t_m, 1) + t_conf = np.take_along_axis(t_main, indices, 1).reshape((66,)) + t_off_x = np.take_along_axis(tensor[66:132].reshape((66,self.out_res_i * self.out_res_i)), indices, 1).reshape((66,)) + t_off_y = np.take_along_axis(tensor[132:198].reshape((66,self.out_res_i * self.out_res_i)), indices, 1).reshape((66,)) + t_off_x = np.floor(res * logit_arr(t_off_x, self.logit_factor) + 0.5) + t_off_y = np.floor(res * logit_arr(t_off_y, self.logit_factor) + 0.5) + t_x = crop_y1 + scale_y * (res * np.floor(t_m / self.out_res_i) / self.out_res + t_off_x) + t_y = crop_x1 + scale_x * (res * np.floor(np.mod(t_m, self.out_res_i)) / self.out_res + t_off_y) + avg_conf = np.average(t_conf) + lms = np.stack([t_x, t_y, t_conf], 1) return (avg_conf, np.array(lms)) def estimate_depth(self, face_info): From 76bd3dcf3aebb9a93713890aeccf36210aa5cc6f Mon Sep 17 00:00:00 2001 From: Emiliana <38952746+emilianavt@users.noreply.github.com> Date: Sun, 9 Aug 2020 19:34:14 +0200 Subject: [PATCH 2/2] Optimized estimate_depth function for a 0.5ms speedup. --- tracker.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/tracker.py b/tracker.py index a9663c7..78c1495 100644 --- a/tracker.py +++ b/tracker.py @@ -708,7 +708,17 @@ class Tracker(): rmat, _ = cv2.Rodrigues(rotation) inverse_rotation = np.linalg.inv(rmat) - pnp_error = 0.0 + t_reference = face_info.face_3d.dot(rmat.transpose()) + t_reference = t_reference + face_info.translation + t_reference = t_reference.dot(self.camera.transpose()) + t_depth = t_reference[:, 2] + t_depth[t_depth == 0] = 0.000001 + t_depth_e = np.expand_dims(t_depth[:],1) + t_reference = t_reference[:] / t_depth_e + pts_3d[0:66] = np.stack([lms[0:66,0], lms[0:66,1], np.ones((66,))], 1) * t_depth_e[0:66] + pts_3d[0:66] = (pts_3d[0:66].dot(self.inverse_camera.transpose()) - face_info.translation).dot(inverse_rotation.transpose()) + pnp_error = np.power(lms[0:17,0:2] - t_reference[0:17,0:2], 2).sum() + pnp_error += np.power(lms[30,0:2] - t_reference[30,0:2], 2).sum() for i, pt in enumerate(face_info.face_3d): if i == 68: # Right eyeball @@ -737,20 +747,16 @@ class Tracker(): d2 = np.linalg.norm(lms[i,0:2] - lms[45,0:2]) d = d1 + d2 pt = (pts_3d[42] * d1 + pts_3d[45] * d2) / d - reference = rmat.dot(pt) - reference = reference + face_info.translation - reference = self.camera.dot(reference) - depth = reference[2] - if i < 17 or i == 30: - reference = reference / depth - e1 = lms[i][0] - reference[0] - e2 = lms[i][1] - reference[1] - pnp_error += e1*e1 + e2*e2 - pt_3d = np.array([lms[i][0] * depth, lms[i][1] * depth, depth], np.float32) - pt_3d = self.inverse_camera.dot(pt_3d) - pt_3d = pt_3d - face_info.translation - pt_3d = inverse_rotation.dot(pt_3d) - pts_3d[i,:] = pt_3d[:] + if i == 66 or i == 67: + reference = rmat.dot(pt) + reference = reference + face_info.translation + reference = self.camera.dot(reference) + depth = reference[2] + pt_3d = np.array([lms[i][0] * depth, lms[i][1] * depth, depth], np.float32) + pt_3d = self.inverse_camera.dot(pt_3d) + pt_3d = pt_3d - face_info.translation + pt_3d = inverse_rotation.dot(pt_3d) + pts_3d[i,:] = pt_3d[:] pnp_error = np.sqrt(pnp_error / (2.0 * image_pts.shape[0])) if pnp_error > 300: