From f793bb270aadb0d331439c5fb47c000dbb9c92b6 Mon Sep 17 00:00:00 2001 From: Cowwan Date: Tue, 6 Feb 2024 12:14:04 -0500 Subject: [PATCH] Added some object-oriented-ness to the EyeTracker class --- eyes.py | 66 ++++++++++++++++++++++++++++++++++++++---------------- tracker.py | 21 +++-------------- 2 files changed, 50 insertions(+), 37 deletions(-) diff --git a/eyes.py b/eyes.py index eee19d0..affd420 100644 --- a/eyes.py +++ b/eyes.py @@ -42,6 +42,12 @@ class EyeTracker(): self.eye_tracking_frames = 0 self.mean = np.float32(np.array([-2.1179, -2.0357, -1.8044])) self.std = np.float32(np.array([0.0171, 0.0175, 0.0174])) + self.faceFrame = None + self.eyeState = None + self.offset = None + self.results = None + self.faceCenter = None + self.faceRadius = None model_base_path = os.path.join(os.path.dirname(__file__), os.path.join("models")) providersList = onnxruntime.capi._pybind_state.get_available_providers() @@ -54,27 +60,31 @@ class EyeTracker(): self.gaze_model = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options, providers=providersList) - def get_eye_state(self,face_frame, lms, offset): - eye_state = [[1.,0.,0.,0.],[1.,0.,0.,0.]] #Array constructed early so it can be used for an early exit + def get_eye_state(self,frame, lms): + self.eyeState = [[1.,0.,0.,0.],[1.,0.,0.,0.]] #Array constructed early so it can be used for an early exit + + lms = self.extract_face(frame, np.array(lms)[:,0:2][:,::-1]) + self.eye_tracking_frames+=1 #I've tried to thread this because they don't need to happen in sequence #but it doesn't do any good because python threads kinda suck #I'd need a persistent right eye and left eye process and that feels excessive - right_eye, REyeInfo = self.prepare_eye(face_frame, lms[36,0:2], lms[39,0:2], False) - left_eye, LEyeInfo = self.prepare_eye(face_frame,lms[42,0:2], lms[45,0:2], True) + right_eye, REyeInfo = self.prepare_eye(lms[36,0:2], lms[39,0:2], False) + left_eye, LEyeInfo = self.prepare_eye(lms[42,0:2], lms[45,0:2], True) if right_eye is None or left_eye is None: return eye_state #Early exit if one of the eyes doesn't have data both_eyes = np.concatenate((right_eye, left_eye)) - results = self.gaze_model.run([], {"input": both_eyes})[0] - eye_state[0] = self.calculateEye(results[0], REyeInfo, offset,0) - eye_state[1] = self.calculateEye(results[1], LEyeInfo, offset,1) - return eye_state + self.results = self.gaze_model.run([], {"input": both_eyes})[0] + self.calculateEye( REyeInfo, 0) + self.calculateEye( LEyeInfo, 1) + return self.eyeState def calculateStandardDeviation(self, conf, eye): + avgRatio = 1/self.eye_tracking_frames self.average_eye_confidence[eye] = (self.average_eye_confidence[eye]*(1-avgRatio))+ (conf * avgRatio) @@ -83,31 +93,30 @@ class EyeTracker(): self.eye_std_dev[eye] = math.sqrt(self.average_eye_confidence_variance[eye]) - def calculateEye(self, results, eye_info , offset, eye): + def calculateEye(self, eye_info, eye): e_x, e_y, scale, reference, angles = eye_info confidenceThreshold = self.average_eye_confidence[eye] - 2 * self.eye_std_dev[eye] - m = results[0].argmax() + m = self.results[eye][0].argmax() x = m // 8 y = m % 8 - p=results[1][x, y] + p=self.results[eye][1][x, y] p = clamp(p, 0.00001, 0.9999) off_x = math.log(p/(1-p)) eye_x = 4.0 *(x + off_x) - p=results[2][x, y] + p=self.results[eye][2][x, y] p = clamp(p, 0.00001, 0.9999) off_y = math.log(p/(1-p)) eye_y = 4.0 * (y + off_y) #I'm proud of this - #it's my "anti-Cali" functionality #if eye movements are below 2 standard deviations of the average the movement is severely limited #the eyes are handled independenly because my testing showed that one eye tended to have a higher condifence than the other - limit = max((results[0][x,y]- 0.5)*(2/confidenceThreshold),0) - if results[0][x,y] < confidenceThreshold: + limit = max((self.results[eye][0][x,y]- 0.5)*(2/confidenceThreshold),0) + if self.results[eye][0][x,y] < confidenceThreshold: Delta = eye_y - self.last_eye_state[eye][0] eye_y = self.last_eye_state[eye][0] + clamp(Delta, -limit, limit) Delta = eye_x - self.last_eye_state[eye][1] @@ -121,11 +130,15 @@ class EyeTracker(): eye_y = e_y + scale[1] * eye_y eye_x, eye_y = rotate(reference, (eye_x, eye_y), -angles) - eye_x, eye_y = (eye_x, eye_y) + offset - self.calculateStandardDeviation(results[0][x,y], eye) - return [1.0, eye_y, eye_x, results[0][x,y]] + eye_x, eye_y = (eye_x, eye_y) + self.offset - def prepare_eye(self,im, outer_pt, inner_pt, flip): + conf = self.results[eye][0][x,y] + self.calculateStandardDeviation(conf, eye) + self.eyeState[eye] = [1.0, eye_y, eye_x, conf] + return + + def prepare_eye(self, outer_pt, inner_pt, flip): + im = self.faceFrame (x1, y1), (x2, y2), center, reference, a = self.corners_to_eye((outer_pt, inner_pt), im.shape[1], im.shape[0]) #rotating an image is expensive and reduces clarity #so I just don't if it's a relatively small angle @@ -154,3 +167,18 @@ class EyeTracker(): upper_left = clamp_to_im(center - radius, w, h) lower_right = clamp_to_im(center + radius, w, h) return upper_left, lower_right, center, c1, a + + + def extract_face(self, frame, lms): + x1, y1 = lms.min(0) + x2, y2 = lms.max(0) + self.faceRadius = np.array([(x2 - x1), (y2 - y1)])*0.6 + self.faceCenter = (np.array((x1, y1)) + np.array((x2, y2))) / 2.0 + w, h, _ = frame.shape + x1, y1 = clamp_to_im(self.faceCenter - self.faceRadius , h, w) + x2, y2 = clamp_to_im(self.faceCenter + self.faceRadius + 1, h, w) + self.offset = np.array((x1, y1)) + lms = (lms[:, 0:2] - self.offset).astype(int) + frame = frame[y1:y2, x1:x2] + self.faceFrame = frame + return lms diff --git a/tracker.py b/tracker.py index 476262a..0cbfa3d 100644 --- a/tracker.py +++ b/tracker.py @@ -11,18 +11,6 @@ import emilianaFeatureExtractor #this file is so much smaller than it used to be, lol #I moved most of the functionality into separate files so they're easier to work with -def extract_face( frame, lms): - x1, y1 = lms.min(0) - x2, y2 = lms.max(0) - radius = np.array([(x2 - x1), (y2 - y1)])*0.6 - center = (np.array((x1, y1)) + np.array((x2, y2))) / 2.0 - w, h, _ = frame.shape - x1, y1 = clamp_to_im(center - radius, h, w) - x2, y2 = clamp_to_im(center + radius + 1, h, w) - offset = np.array((x1, y1)) - lms = (lms[:, 0:2] - offset).astype(int) - frame = frame[y1:y2, x1:x2] - return frame, lms, offset, center, radius def clamp_to_im(pt, w, h): #8 times per frame, but that only accounts for 0.005ms x=max(pt[0],0) @@ -49,8 +37,6 @@ class Tracker(): self.mean = np.float32(np.array([-2.1179, -2.0357, -1.8044])) self.std = np.float32(np.array([0.0171, 0.0175, 0.0174])) - self.camera = np.array([[width, 0, width/2], [0, width, height/2], [0, 0, 1]], np.float32) - self.inverse_camera = np.linalg.inv(self.camera) self.width = width self.height = height @@ -133,9 +119,8 @@ class Tracker(): if conf < self.threshold: return self.early_exit("Confidence below threshold", start) - #lms is short for landmarks - face_frame, face_lms, face_offset,face_center, face_radius = extract_face(frame, np.array(lms)[:,0:2][:,::-1]) - eye_state = self.EyeTracker.get_eye_state(face_frame, face_lms, face_offset) + eye_state = self.EyeTracker.get_eye_state(frame, lms) + self.face_info[0].update((conf, (lms, eye_state)), np.array(lms)[:, 0:2].mean(0)) @@ -155,7 +140,7 @@ class Tracker(): duration_pnp += 1000 * (time.perf_counter() - start_pnp) duration = (time.perf_counter() - start) * 1000 self.messageQueue.put(f"Took {duration:.2f}ms (detect: {duration_fd:.2f}ms, crop: {duration_pp:.2f}ms, track: {duration_model:.2f}ms, 3D points: {duration_pnp:.2f}ms)") - return face_info, face_center, face_radius + return face_info, self.EyeTracker.faceCenter, self.EyeTracker.faceRadius #Combined multiple failures into one catch all exit return self.early_exit("Face info not valid", start)