From 4675b9a7224ffcf7be0b1540cb0e6960bc9c1e1f Mon Sep 17 00:00:00 2001 From: Cowwan Date: Tue, 6 Feb 2024 13:27:20 -0500 Subject: [PATCH] fixed numpy cpu usage made eye code cleaner --- eyes.py | 43 +++++++++++++++++++++---------------------- facetracker.py | 1 + 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/eyes.py b/eyes.py index 2fdb89f..5450a14 100644 --- a/eyes.py +++ b/eyes.py @@ -28,7 +28,7 @@ def clamp (value, minimum, maxium): return max(min(value,maxium),minimum) def rotate_image(image, a, center): #twice per frame, 0.2ms - 0.25ms each, improved to 0.15ms - 0.25ms - M = cv2.getRotationMatrix2D(center, math.degrees(a), 1.0) + M = cv2.getRotationMatrix2D((float(center[0]), float(center[1])), math.degrees(a), 1.0) (h, w) = image.shape[:2] image = cv2.warpAffine(image, M, (w, h), flags = cv2.INTER_CUBIC) return image @@ -53,14 +53,17 @@ class Eye(): self.averageEyeConfidence = 1. self.averageEyeConfidenceVariance = 0 self.lastEyeState = [0.,0.] + self.innerPoint = None + self.outerPoint = None - def prepare_eye(self, faceFrame, outer_pt, inner_pt): + def prepare_eye(self, faceFrame): + self.state = [1.,0.,0.,0.] im = faceFrame - (x1, y1), (x2, y2), center, reference, a = self.corners_to_eye((outer_pt, inner_pt), im.shape[1], im.shape[0]) + (x1, y1), (x2, y2), a = self.corners_to_eye( im.shape[1], im.shape[0]) #rotating an image is expensive and reduces clarity #so I just don't if it's a relatively small angle if math.degrees(a) > 7.5 and math.degrees(a) < 352.5: - im = rotate_image(im, a, reference) + im = rotate_image(im, a, self.outerPoint) im = im[int(y1):int(y2), int(x1):int(x2)] if np.prod(im.shape) < 1: @@ -72,11 +75,12 @@ class Eye(): im = im.astype(np.float32) * self.std + self.mean im = np.expand_dims(im, 0) self.image = np.transpose(im, (0,3,2,1)) - self.info = [x1, y1, scale, reference, a] + self.info = [x1, y1, scale, self.outerPoint, a] return - def corners_to_eye(self,corners, w, h): - c1, c2 = np.array(corners, dtype=float) + def corners_to_eye(self, w, h): + c1 = np.array(self.outerPoint) + c2 = np.array(self.innerPoint) a = math.atan2(*(c2 - c1)[::-1]) % (2*math.pi) c2 = rotate(c1, c2, a) center = (c1 + c2) / 2.0 @@ -84,7 +88,7 @@ class Eye(): radius = [radius * 0.7, radius * 0.6] upper_left = clamp_to_im(center - radius, w, h) lower_right = clamp_to_im(center + radius, w, h) - return upper_left, lower_right, center, c1, a + return upper_left, lower_right, a def calculateEye(self): @@ -149,7 +153,6 @@ class EyeTracker(): self.mean = np.float32(np.array([-2.1179, -2.0357, -1.8044])) self.std = np.float32(np.array([0.0171, 0.0175, 0.0174])) self.faceFrame = None - self.eyeState = None self.offset = None self.faceCenter = None self.faceRadius = None @@ -164,33 +167,30 @@ class EyeTracker(): options.log_severity_level = 3 self.gaze_model = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options, providers=providersList) - def get_eye_state(self,frame, lms): - self.eyeState = [[1.,0.,0.,0.],[1.,0.,0.,0.]] #Array constructed early so it can be used for an early exit lms = self.extract_face(frame, np.array(lms)[:,0:2][:,::-1]) self.rightEye.offset = self.offset self.leftEye.offset = self.offset + self.rightEye.innerPoint = lms[39,0:2] + self.rightEye.outerPoint = lms[36,0:2] + self.leftEye.innerPoint = lms[45,0:2] + self.leftEye.outerPoint = lms[42,0:2] - - self.rightEye.prepare_eye(self.faceFrame, lms[36,0:2], lms[39,0:2]) - self.leftEye.prepare_eye(self.faceFrame, lms[42,0:2], lms[45,0:2]) + self.rightEye.prepare_eye(self.faceFrame) + self.leftEye.prepare_eye(self.faceFrame) if self.rightEye.image is None or self.leftEye.image is None: - return eye_state #Early exit if one of the eyes doesn't have data + return [[1.,0.,0.,0.],[1.,0.,0.,0.]] #Early exit if one of the eyes doesn't have data both_eyes = np.concatenate((self.rightEye.image, self.leftEye.image)) self.rightEye.results, self.leftEye.results = self.gaze_model.run([], {"input": both_eyes})[0] - self.rightEye.calculateEye() self.leftEye.calculateEye() - self.eyeState = [self.rightEye.state, self.leftEye.state] - - return self.eyeState - + return [self.rightEye.state, self.leftEye.state] def extract_face(self, frame, lms): x1, y1 = lms.min(0) @@ -202,6 +202,5 @@ class EyeTracker(): x2, y2 = clamp_to_im(self.faceCenter + self.faceRadius + 1, h, w) self.offset = np.array((x1, y1)) lms = (lms[:, 0:2] - self.offset).astype(int) - frame = frame[y1:y2, x1:x2] - self.faceFrame = frame + self.faceFrame = frame[y1:y2, x1:x2] return lms diff --git a/facetracker.py b/facetracker.py index faf7e8e..d23b65a 100644 --- a/facetracker.py +++ b/facetracker.py @@ -3,6 +3,7 @@ import os #limits the threads used by numpy to 1 #setting this and the landmarks model to use 1 means the cpu usage is pretty low +os.environ["OMP_NUM_THREADS"] = str(1) import argparse import traceback import threading