copilot/OpenSeeFace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fixed gaze tracking issues introduced recently.

Browse source
This commit is contained in:
Emiliana 2020-09-05 15:05:32 +02:00
parent 0c91cab5c6
commit 1aca1bee66
1 changed files with 23 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

36
tracker.py
View file

@ -512,7 +512,7 @@ class Tracker():
if threshold is None: if threshold is None:
threshold = 0.6 threshold = 0.6
if model_type < 0: if model_type < 0:
threshold = 0.84 threshold = 0.87
self.retinaface = RetinaFaceDetector(model_path=os.path.join(model_base_path, "retinaface_640x640_opt.onnx"), json_path=os.path.join(model_base_path, "priorbox_640x640.json"), threads=max(max_threads,4), top_k=max_faces, res=(640, 640)) self.retinaface = RetinaFaceDetector(model_path=os.path.join(model_base_path, "retinaface_640x640_opt.onnx"), json_path=os.path.join(model_base_path, "priorbox_640x640.json"), threads=max(max_threads,4), top_k=max_faces, res=(640, 640))
self.retinaface_scan = RetinaFaceDetector(model_path=os.path.join(model_base_path, "retinaface_640x640_opt.onnx"), json_path=os.path.join(model_base_path, "priorbox_640x640.json"), threads=2, top_k=max_faces, res=(640, 640)) self.retinaface_scan = RetinaFaceDetector(model_path=os.path.join(model_base_path, "retinaface_640x640_opt.onnx"), json_path=os.path.join(model_base_path, "priorbox_640x640.json"), threads=2, top_k=max_faces, res=(640, 640))
@ -865,21 +865,28 @@ class Tracker():
return upper_left, lower_right, center, radius, c1, a return upper_left, lower_right, center, radius, c1, a
def prepare_eye(self, frame, full_frame, lms, flip): def prepare_eye(self, frame, full_frame, lms, flip):
transform = SimilarityTransform() outer_pt = tuple(lms[0])
transform.estimate(lms, np.array([[0,16], [32,16]])) inner_pt = tuple(lms[1])
M = transform.params[0:2, :] h, w, _ = frame.shape
im = cv2.warpAffine(frame, M, (32, 32), 0) (x1, y1), (x2, y2), center, radius, reference, a = self.corners_to_eye((outer_pt, inner_pt), w, h, flip)
im = rotate_image(frame[:, :, ::], a, reference)
im = im[int(y1):int(y2), int(x1):int(x2),:]
if np.prod(im.shape) < 1:
return None, None, None, None, None, None
if flip: if flip:
im = cv2.flip(im, 1) im = cv2.flip(im, 1)
scale = np.array([(x2 - x1), (y2 - y1)]) / 32.
im = cv2.resize(im, (32, 32), interpolation=cv2.INTER_LINEAR)
#im = self.equalize(im)
if self.debug_gaze: if self.debug_gaze:
if not flip: if not flip:
full_frame[0:32, 0:32] = im full_frame[0:32, 0:32] = im
else: else:
full_frame[0:32, 32:64] = im full_frame[0:32, 32:64] = im
im = im.astype(np.float32)[:,:,::-1] * self.mean_32 + self.std_32 im = im.astype(np.float32)[:,:,::-1] * self.std_32 + self.mean_32
im = np.expand_dims(im, 0) im = np.expand_dims(im, 0)
im = np.transpose(im, (0,3,2,1)) im = np.transpose(im, (0,3,2,1))
return im, np.linalg.inv(transform.params) return im, x1, y1, scale, reference, a
def extract_face(self, frame, lms): def extract_face(self, frame, lms):
lms = np.array(lms)[:,0:2][:,::-1] lms = np.array(lms)[:,0:2][:,::-1]
@ -905,10 +912,10 @@ class Tracker():
e_y = [0,0] e_y = [0,0]
scale = [0,0] scale = [0,0]
reference = [None, None] reference = [None, None]
M_inv = [np.identity(3), np.identity(3)] angles = [0, 0]
face_frame, lms, offset = self.extract_face(frame, lms) face_frame, lms, offset = self.extract_face(frame, lms)
(right_eye, M_inv[0]) = self.prepare_eye(face_frame, frame, np.array([lms[36,0:2], lms[39,0:2]]), False) (right_eye, e_x[0], e_y[0], scale[0], reference[0], angles[0]) = self.prepare_eye(face_frame, frame, np.array([lms[36,0:2], lms[39,0:2]]), False)
(left_eye, M_inv[1]) = self.prepare_eye(face_frame, frame, np.array([lms[42,0:2], lms[45,0:2]]), True) (left_eye, e_x[1], e_y[1], scale[1], reference[1], angles[1]) = self.prepare_eye(face_frame, frame, np.array([lms[42,0:2], lms[45,0:2]]), True)
if right_eye is None or left_eye is None: if right_eye is None or left_eye is None:
return [(1.0, 0.0, 0.0, 0.0), (1.0, 0.0, 0.0, 0.0)] return [(1.0, 0.0, 0.0, 0.0), (1.0, 0.0, 0.0, 0.0)]
both_eyes = np.concatenate((right_eye, left_eye)) both_eyes = np.concatenate((right_eye, left_eye))
@ -949,10 +956,13 @@ class Tracker():
frame[int(eye_y+1), 32+int(eye_x+1)] = (0, 0, 255) frame[int(eye_y+1), 32+int(eye_x+1)] = (0, 0, 255)
frame[int(eye_y), 32+int(eye_x+1)] = (0, 0, 255) frame[int(eye_y), 32+int(eye_x+1)] = (0, 0, 255)
if i == 1: if i == 0:
eye_x = 32. - eye_x eye_x = e_x[i] + scale[i][0] * eye_x
else:
eye_x = e_x[i] + scale[i][0] * (32. - eye_x)
eye_y = e_y[i] + scale[i][1] * eye_y
eye_x, eye_y = rotate(reference[i], (eye_x, eye_y), -angles[i])
eye_x, eye_y, w = M_inv[i].dot(np.array([eye_x, eye_y, 1]))
eye_x = eye_x + offset[0] eye_x = eye_x + offset[0]
eye_y = eye_y + offset[1] eye_y = eye_y + offset[1]
eye_state.append([open[i], eye_y, eye_x, conf]) eye_state.append([open[i], eye_y, eye_x, conf])