diff --git a/landmarks.py b/landmarks.py index e038c18..aa6fad1 100644 --- a/landmarks.py +++ b/landmarks.py @@ -21,6 +21,35 @@ def matrix_to_quaternion(m): q = [m[1,2]-m[2,1], m[2,0]-m[0,2], m[0,1]-m[1,0], t] return np.array(q) / (math.sqrt(t)* 0.5) +def landmarks(tensor, crop_info): + crop_x1, crop_y1, scale_x, scale_y = crop_info + + t_main = tensor[0:66].reshape((66,784)) + t_m = t_main.argmax(1) + indices = np.expand_dims(t_m, 1) + + t_off_x = np.take_along_axis(tensor[66:132].reshape((66,784)), indices, 1).reshape((66,)) + p = np.clip(t_off_x, 0.0000001, 0.9999999) + t_off_x = 13.9375 * np.log(p / (1 - p)) + t_x = crop_y1 + scale_y * (223. * np.floor(t_m / 28) / 27.+ t_off_x) + + t_off_y = np.take_along_axis(tensor[132:198].reshape((66,784)), indices, 1).reshape((66,)) + p = np.clip(t_off_y, 0.0000001, 0.9999999) + t_off_y = 13.9375 * np.log(p / (1 - p)) + t_y = crop_x1 + scale_x * (223. * np.floor(np.mod(t_m, 28)) / 27. + t_off_y) + + t_conf = np.take_along_axis(t_main, indices, 1).reshape((66,)) + lms = np.stack([t_x, t_y, t_conf], 1) + lms[np.isnan(lms).any(axis=1)] = np.array([0.,0.,0.], dtype=np.float32) + return (np.average(t_conf), lms) + +def run( models, crop, crop_info): + + output = models.landmarks.run([], {"input": crop})[0][0] + confidence, lms = landmarks(output, crop_info) + return (confidence, lms) + + #I haven't touched much here #not because I didn't try to improve things #but because I caused problems when I did @@ -31,120 +60,18 @@ class Landmarks(): self.inverse_camera = np.linalg.inv(self.camera) self.width = width self.height = height - self.confidence = 0 - - self.face_3d = np.array([ - [ 0.4551769692672 , 0.300895790030204, -0.764429433974752], - [ 0.448998827123556, 0.166995837790733, -0.765143004071253], - [ 0.437431554952677, 0.022655479179981, -0.739267175112735], - [ 0.415033422928434, -0.088941454648772, -0.747947437846473], - [ 0.389123587370091, -0.232380029794684, -0.704788385327458], - [ 0.334630113904382, -0.361265387599081, -0.615587579236862], - [ 0.263725112132858, -0.460009725616771, -0.491479221041573], - [ 0.16241621322721 , -0.558037146073869, -0.339445180872282], - [ 0. , -0.621079019321682, -0.287294770748887], - [-0.16241621322721 , -0.558037146073869, -0.339445180872282], - [-0.263725112132858, -0.460009725616771, -0.491479221041573], - [-0.334630113904382, -0.361265387599081, -0.615587579236862], - [-0.389123587370091, -0.232380029794684, -0.704788385327458], - [-0.415033422928434, -0.088941454648772, -0.747947437846473], - [-0.437431554952677, 0.022655479179981, -0.739267175112735], - [-0.448998827123556, 0.166995837790733, -0.765143004071253], - [-0.4551769692672 , 0.300895790030204, -0.764429433974752], - [ 0.385529968662985, 0.402800553948697, -0.310031082540741], - [ 0.322196658344302, 0.464439136821772, -0.250558059367669], - [ 0.25409760441282 , 0.46420381416882 , -0.208177722146526], - [ 0.186875436782135, 0.44706071961879 , -0.145299823706503], - [ 0.120880983543622, 0.423566314072968, -0.110757158774771], - [-0.120880983543622, 0.423566314072968, -0.110757158774771], - [-0.186875436782135, 0.44706071961879 , -0.145299823706503], - [-0.25409760441282 , 0.46420381416882 , -0.208177722146526], - [-0.322196658344302, 0.464439136821772, -0.250558059367669], - [-0.385529968662985, 0.402800553948697, -0.310031082540741], - [ 0. , 0.293332603215811, -0.137582088779393], - [ 0. , 0.194828701837823, -0.069158109325951], - [ 0. , 0.103844017393155, -0.009151819844964], - [ 0. , 0. , 0. ], - [ 0.080626352317973, -0.041276068128093, -0.134161035564826], - [ 0.046439347377934, -0.057675223874769, -0.102990627164664], - [ 0. , -0.068753126205604, -0.090545348482397], - [-0.046439347377934, -0.057675223874769, -0.102990627164664], - [-0.080626352317973, -0.041276068128093, -0.134161035564826], - [ 0.315905195966084, 0.298337502555443, -0.285107407636464], - [ 0.275252345439353, 0.312721904921771, -0.244558251170671], - [ 0.176394511553111, 0.311907184376107, -0.219205360345231], - [ 0.131229723798772, 0.284447361805627, -0.234239149487417], - [ 0.184124948330084, 0.260179585304867, -0.226590776513707], - [ 0.279433549294448, 0.267363071770222, -0.248441437111633], - [-0.131229723798772, 0.284447361805627, -0.234239149487417], - [-0.176394511553111, 0.311907184376107, -0.219205360345231], - [-0.275252345439353, 0.312721904921771, -0.244558251170671], - [-0.315905195966084, 0.298337502555443, -0.285107407636464], - [-0.279433549294448, 0.267363071770222, -0.248441437111633], - [-0.184124948330084, 0.260179585304867, -0.226590776513707], - [ 0.121155252430729, -0.208988660580347, -0.160606287940521], - [ 0.041356305910044, -0.194484199722098, -0.096159882202821], - [ 0. , -0.205180167345702, -0.083299217789729], - [-0.041356305910044, -0.194484199722098, -0.096159882202821], - [-0.121155252430729, -0.208988660580347, -0.160606287940521], - [-0.132325402795928, -0.290857984604968, -0.187067868218105], - [-0.064137791831655, -0.325377847425684, -0.158924039726607], - [ 0. , -0.343742581679188, -0.113925986025684], - [ 0.064137791831655, -0.325377847425684, -0.158924039726607], - [ 0.132325402795928, -0.290857984604968, -0.187067868218105], - [ 0.181481567104525, -0.243239316141725, -0.231284988892766], - [ 0.083999507750469, -0.239717753728704, -0.155256465640701], - [ 0. , -0.256058040176369, -0.0950619498899 ], - [-0.083999507750469, -0.239717753728704, -0.155256465640701], - [-0.181481567104525, -0.243239316141725, -0.231284988892766], - [-0.074036069749345, -0.250689938345682, -0.177346470406188], - [ 0. , -0.264945854681568, -0.112349967428413], - [ 0.074036069749345, -0.250689938345682, -0.177346470406188], - # Pupils and eyeball centers - [ 0.257990002632141, 0.276080012321472, -0.219998998939991], - [-0.257990002632141, 0.276080012321472, -0.219998998939991], - [ 0.257990002632141, 0.276080012321472, -0.324570998549461], - [-0.257990002632141, 0.276080012321472, -0.324570998549461] - ], np.float32) - - self.contour_pts = [0,1,8,15,16,27,28,29,30,31,32,33,34,35] self.fail_count = 0 - def run(self, models, crop, crop_info): - output = models.landmarks.run([], {"input": crop})[0][0] - self.confidence, lms = self.landmarks(output, crop_info) - return ( lms) - - def landmarks(self, tensor, crop_info): - crop_x1, crop_y1, scale_x, scale_y = crop_info - - t_main = tensor[0:66].reshape((66,784)) - t_m = t_main.argmax(1) - indices = np.expand_dims(t_m, 1) - - t_off_x = np.take_along_axis(tensor[66:132].reshape((66,784)), indices, 1).reshape((66,)) - p = np.clip(t_off_x, 0.0000001, 0.9999999) - t_off_x = 13.9375 * np.log(p / (1 - p)) - t_x = crop_y1 + scale_y * (223. * np.floor(t_m / 28) / 27.+ t_off_x) - - t_off_y = np.take_along_axis(tensor[132:198].reshape((66,784)), indices, 1).reshape((66,)) - p = np.clip(t_off_y, 0.0000001, 0.9999999) - t_off_y = 13.9375 * np.log(p / (1 - p)) - t_y = crop_x1 + scale_x * (223. * np.floor(np.mod(t_m, 28)) / 27. + t_off_y) - - t_conf = np.take_along_axis(t_main, indices, 1).reshape((66,)) - lms = np.stack([t_x, t_y, t_conf], 1) - lms[np.isnan(lms).any(axis=1)] = np.array([0.,0.,0.], dtype=np.float32) - return (np.average(t_conf), lms) #I really need to break this guy up into smaller parts def estimate_depth(self, face_info): + contour_pts = [0,1,8,15,16,27,28,29,30,31,32,33,34,35] lms = np.concatenate((face_info.lms, np.array([[face_info.eye_state[0][1], face_info.eye_state[0][2], face_info.eye_state[0][3]], [face_info.eye_state[1][1], face_info.eye_state[1][2], face_info.eye_state[1][3]]], np.float32)), 0) - image_pts = np.array(lms)[self.contour_pts, 0:2] + image_pts = np.array(lms)[contour_pts, 0:2] success = False if face_info.rotation is not None: @@ -227,8 +154,7 @@ class Landmarks(): self.fail_count += 1 if self.fail_count > 5: # Something went wrong with adjusting the 3D model - print(f"Detected anomaly when 3D fitting face {face_info.id}. Resetting.") - face_info.face_3d = self.face_3d + print(f"Detected anomaly when 3D fitting face {0}. Resetting.") face_info.rotation = None face_info.translation = np.array([0.0, 0.0, 0.0], np.float32) face_info.update_counts = np.zeros((66,2)) diff --git a/tracker.py b/tracker.py index 63eafee..b1dff69 100644 --- a/tracker.py +++ b/tracker.py @@ -130,14 +130,14 @@ class Tracker(): if crop is None: return self.early_exit("No valid crops", start) start_model = time.perf_counter() - lms = self.Landmarks.run(self.model, crop, crop_info) + confidence, lms = landmarks.run(self.model, crop, crop_info) #Early exit if below confidence threshold - if self.Landmarks.confidence < self.threshold: + if confidence < self.threshold: return self.early_exit("Confidence below threshold", start) eye_state = self.EyeTracker.get_eye_state(self.model, frame, lms) - self.face_info.update((self.Landmarks.confidence, (lms, eye_state)), np.array(lms)[:, 0:2].mean(0)) + self.face_info.update((confidence, (lms, eye_state)), np.array(lms)[:, 0:2].mean(0)) duration_model = 1000 * (time.perf_counter() - start_model) start_pnp = time.perf_counter()