diff --git a/Unity/OpenSee.cs b/Unity/OpenSee.cs index be1a80c..fae3270 100644 --- a/Unity/OpenSee.cs +++ b/Unity/OpenSee.cs @@ -101,13 +101,13 @@ public class OpenSee : MonoBehaviour { rawEuler = readVector3(b, ref o); rotation = rawEuler; - rotation.z = (rotation.z + 90) % 360; + rotation.z = (rotation.z - 90) % 360; rotation.x = -(rotation.x + 180) % 360; float x = readFloat(b, ref o); float y = readFloat(b, ref o); float z = readFloat(b, ref o); - translation = new Vector3(y, -x, z); + translation = new Vector3(-y, x, -z); for (int i = 0; i < nPoints; i++) { confidence[i] = readFloat(b, ref o); diff --git a/facetracker.py b/facetracker.py index 3582af8..a6c399c 100644 --- a/facetracker.py +++ b/facetracker.py @@ -13,7 +13,7 @@ if os.name == 'nt': else: parser.add_argument("-W", "--width", type=int, help="Set raw RGB width", default=640) parser.add_argument("-H", "--height", type=int, help="Set raw RGB height", default=360) -parser.add_argument("-c", "--capture", help="Set camera ID (0, 1...) or video file", default=0) +parser.add_argument("-c", "--capture", help="Set camera ID (0, 1...) or video file", default="0") parser.add_argument("-m", "--max-threads", type=int, help="Set the maximum number of threads", default=1) parser.add_argument("-t", "--threshold", type=float, help="Set minimum confidence threshold for face detection", default=0.4) parser.add_argument("-v", "--visualize", type=int, help="Set this to 1 to visualize the tracking or to 2 to add numbers to the point display", default=0) @@ -79,7 +79,7 @@ frame_count = 0 if args.log_data != "": log = open(args.log_data, "w") - log.write("Frame,Time,Face,AverageConfidence,Success3D,PnPError,RotationQuat.X,RotationQuat.Y,RotationQuat.Z,RotationQuat.W,RVec.X,RVec.Y,RVec.Z,TVec.X,TVec.Y,TVec.Z") + log.write("Frame,Time,Face,RightOpen,LeftOpen,AverageConfidence,Success3D,PnPError,RotationQuat.X,RotationQuat.Y,RotationQuat.Z,RotationQuat.W,RVec.X,RVec.Y,RVec.Z,TVec.X,TVec.Y,TVec.Z") for i in range(66): log.write(f",Landmark[{i}].X,Landmark[{i}].Y,Landmark[{i}].Confidence") for i in range(66): @@ -108,16 +108,18 @@ try: if not args.video_out is None: out = cv2.VideoWriter(args.video_out, cv2.VideoWriter_fourcc('M','J','P','G'), 24, (width,height)) - inference_start = time.time() + inference_start = time.perf_counter() faces = tracker.predict(frame) if len(faces) > 0: - tracking_time += (time.time() - inference_start) / len(faces) + tracking_time += (time.perf_counter() - inference_start) / len(faces) tracking_frames += 1 packet = bytearray() detected = False - for face_num, (conf, lms, success_3d, pnp_error, quaternion, euler, rotation, translation, pts_3d, (right_open, left_open)) in enumerate(faces): # TODO: eye_state + for face_num, (conf, lms, success_3d, pnp_error, quaternion, euler, rotation, translation, pts_3d, (right_open, left_open), (face_y, face_x, face_height, face_width)) in enumerate(faces): + left_state = "O" if left_open > 0.5 else "-" + right_state = "O" if right_open > 0.5 else "-" if args.silent == 0: - print(f"Confidence: {conf:.4f} / 3D fitting error: {pnp_error:.4f}") + print(f"Confidence: {conf:.4f} / 3D fitting error: {pnp_error:.4f} / Eyes: {left_state}, {right_state}") detected = True if not success_3d: pts_3d = np.zeros((70, 3), np.float32) @@ -133,11 +135,11 @@ try: packet.extend(bytearray(struct.pack("f", euler[0]))) packet.extend(bytearray(struct.pack("f", euler[1]))) packet.extend(bytearray(struct.pack("f", euler[2]))) - packet.extend(bytearray(struct.pack("f", translation[0][0]))) - packet.extend(bytearray(struct.pack("f", translation[0][1]))) - packet.extend(bytearray(struct.pack("f", translation[0][2]))) + packet.extend(bytearray(struct.pack("f", translation[0]))) + packet.extend(bytearray(struct.pack("f", translation[1]))) + packet.extend(bytearray(struct.pack("f", translation[2]))) if not log is None: - log.write(f"{frame_count},{now},{face_num},{conf},{success_3d},{pnp_error},{quaternion[0]},{quaternion[1]},{quaternion[2]},{quaternion[3]},{rotation[0][0]},{rotation[0][1]},{rotation[0][2]},{translation[0][0]},{translation[0][1]},{translation[0][2]}") + log.write(f"{frame_count},{now},{face_num},{right_open},{left_open},{conf},{success_3d},{pnp_error},{quaternion[0]},{quaternion[1]},{quaternion[2]},{quaternion[3]},{rotation[0][0]},{rotation[0][1]},{rotation[0][2]},{translation[0][0]},{translation[0][1]},{translation[0][2]}") for (x,y,c) in lms: packet.extend(bytearray(struct.pack("f", c))) for pt_num, (x,y,c) in enumerate(lms): @@ -194,7 +196,7 @@ try: out.write(frame) if args.visualize != 0: - cv2.imshow('Frame', frame) + cv2.imshow('OpenSeeFace Visualization', frame) if cv2.waitKey(1) & 0xFF == ord('q'): break except KeyboardInterrupt: diff --git a/models/snv2_opt.onnx b/models/snv2_opt.onnx index fa2db0f..9114395 100644 Binary files a/models/snv2_opt.onnx and b/models/snv2_opt.onnx differ diff --git a/models/snv2_opt_b.onnx b/models/snv2_opt_b.onnx index 8a759f0..735b2a7 100644 Binary files a/models/snv2_opt_b.onnx and b/models/snv2_opt_b.onnx differ diff --git a/models/snv2_opt_fast.onnx b/models/snv2_opt_fast.onnx index bb7187b..f318040 100644 Binary files a/models/snv2_opt_fast.onnx and b/models/snv2_opt_fast.onnx differ diff --git a/tracker.py b/tracker.py index ff540ad..0736462 100644 --- a/tracker.py +++ b/tracker.py @@ -1,5 +1,6 @@ import os import numpy as np +import math import cv2 import onnxruntime import time @@ -218,7 +219,7 @@ class Tracker(): [-0.25799, 0.27608, -0.16923], [0.25799, 0.27608, -0.24967], [-0.25799, 0.27608, -0.24967], - ], np.float32) * np.array([1.0, 1.0, 1.15]) + ], np.float32) * np.array([1.0, 1.0, 1.3]) self.contour = np.empty((21 if self.high_quality_3d else 18, 3)) self.contour[0:17] = self.face_3d[0:17] @@ -256,23 +257,10 @@ class Tracker(): y = m % 28 conf = float(tensor[i][x,y]) avg_conf = avg_conf + conf - - # Bugfix for heatmaps being offset during training (models need retraining, but this seems to be working fine somehow) - x += 2 - y += 2 - if x >= 28: - x = 27 - if y >= 28: - y = 27 - - off_x = 0 - off_y = 0 - if self.model_type > 0: - off_x = self.res * ((1. * logit(tensor[66 + i][y, x])) - 0.0) - off_y = self.res * ((1. * logit(tensor[66 * 2 + i][y, x])) - 0.0) - else: - off_x = self.res * ((2. * float(tensor[66 + i][y, x])) - 1.0) - off_y = self.res * ((2. * float(tensor[66 * 2 + i][y, x])) - 1.0) + off_x = self.res * ((1. * logit(tensor[66 + i][x, y])) - 0.0) + off_y = self.res * ((1. * logit(tensor[66 * 2 + i][x, y])) - 0.0) + off_x = math.floor(off_x + 0.5) + off_y = math.floor(off_y + 0.5) lm_x = crop_y1 + scale_y * (self.res * (float(x) / 28.) + off_x) lm_y = crop_x1 + scale_x * (self.res * (float(y) / 28.) + off_y) lms.append((lm_x,lm_y,conf)) @@ -291,19 +279,19 @@ class Tracker(): success = False if self.max_faces < 2 and not self.rotation is None: - success, self.rotation, self.translation, inliers = cv2.solvePnPRansac(self.contour, image_pts, self.camera, self.dist_coeffs, useExtrinsicGuess=True, rvec=np.transpose(self.rotation), tvec=np.transpose(self.translation), flags=cv2.SOLVEPNP_ITERATIVE) + success, self.rotation, self.translation = cv2.solvePnP(self.contour, image_pts, self.camera, self.dist_coeffs, useExtrinsicGuess=True, rvec=np.transpose(self.rotation), tvec=np.transpose(self.translation), flags=cv2.SOLVEPNP_ITERATIVE) else: - rvec = np.array([-180, 0, -90], np.float32) + rvec = np.array([0, 0, 0], np.float32) tvec = np.array([0, 0, 0], np.float32) - success, self.rotation, self.translation, inliers = cv2.solvePnPRansac(self.contour, image_pts, self.camera, self.dist_coeffs, useExtrinsicGuess=True, rvec=rvec, tvec=tvec, flags=cv2.SOLVEPNP_ITERATIVE) + success, self.rotation, self.translation = cv2.solvePnP(self.contour, image_pts, self.camera, self.dist_coeffs, useExtrinsicGuess=True, rvec=rvec, tvec=tvec, flags=cv2.SOLVEPNP_ITERATIVE) rotation = self.rotation translation = self.translation pts_3d = np.zeros((70,3), np.float32) if not success: - self.rotation = np.array([[0.0, 0.0, 0.0]], np.float32) - self.translation = np.array([[0.0, 0.0, 0.0]], np.float32) + self.rotation = np.array([0.0, 0.0, 0.0], np.float32) + self.translation = np.array([0.0, 0.0, 0.0], np.float32) return False, np.zeros(4), np.zeros(3), 99999., pts_3d, lms else: self.rotation = np.transpose(self.rotation) @@ -325,7 +313,7 @@ class Tracker(): continue reference = rmat.dot(pt) reference = reference + self.translation - reference = self.camera.dot(reference[0]) + reference = self.camera.dot(reference) depth = reference[2] if i < 17 or i == 30: reference = reference / depth @@ -335,7 +323,7 @@ class Tracker(): pt_3d = np.array([lms[i][0] * depth, lms[i][1] * depth, depth], np.float32) pt_3d = self.inverse_camera.dot(pt_3d) pt_3d = pt_3d - self.translation - pt_3d = inverse_rotation.dot(pt_3d[0]) + pt_3d = inverse_rotation.dot(pt_3d) pts_3d[i,:] = pt_3d[:] euler = cv2.RQDecomp3x3(rmat)[0] @@ -402,7 +390,7 @@ class Tracker(): return eye_state def predict(self, frame, additional_faces=[]): - start = time.time() + start = time.perf_counter() im = frame duration_fd = 0.0 @@ -416,16 +404,16 @@ class Tracker(): self.wait_count += 1 if self.detected < self.max_faces: if self.detected == 0 or self.wait_count >= self.scan_every: - start_fd = time.time() + start_fd = time.perf_counter() gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) new_faces.extend(list(self.faceCascade.detectMultiScale(gray, 1.3, 4, 0 | cv2.CASCADE_SCALE_IMAGE, (50, 50)))) - duration_fd = 1000 * (time.time() - start_fd) + duration_fd = 1000 * (time.perf_counter() - start_fd) self.wait_count = 0 else: self.wait_count = 0 if len(new_faces) < 1: - duration = (time.time() - start) * 1000 + duration = (time.perf_counter() - start) * 1000 if not self.silent: print(f"Took {duration:.2f}ms") return [] @@ -434,16 +422,11 @@ class Tracker(): crop_info = [] num_crops = 0 for j, (x,y,w,h) in enumerate(new_faces): - if self.model_type < 2: - crop_x1 = x - int(w * 0.2) - crop_y1 = y - int(h * 0.25) - crop_x2 = x + w + int(w * 0.2) - crop_y2 = y + h + int(h * 0.25) - else: - crop_x1 = x - int(w * 0.1) - crop_y1 = y - int(h * 0.1) - crop_x2 = x + w + int(w * 0.1) - crop_y2 = y + h + int(h * 0.1) + crop_x1 = x - int(w * 0.1) + crop_y1 = y - int(h * 0.125) + crop_x2 = x + w + int(w * 0.1) + crop_y2 = y + h + int(h * 0.125) + crop_x1, crop_y1 = clamp_to_im((crop_x1, crop_y1), self.width, self.height) crop_x2, crop_y2 = clamp_to_im((crop_x2, crop_y2), self.width, self.height) @@ -453,16 +436,16 @@ class Tracker(): if crop_x2 - crop_x1 < 4 or crop_y2 - crop_y1 < 4: continue - start_pp = time.time() + start_pp = time.perf_counter() crop = self.preprocess(im, (crop_x1, crop_y1, crop_x2, crop_y2)) - duration_pp += 1000 * (time.time() - start_pp) + duration_pp += 1000 * (time.perf_counter() - start_pp) crops.append(crop) crop_info.append((crop_x1, crop_y1, scale_x, scale_y)) num_crops += 1 groups = group_rects(new_faces) - start_model = time.time() + start_model = time.perf_counter() outputs = {} if num_crops == 1: output = self.session.run([], {self.input_name: crops[0]})[0] @@ -499,7 +482,7 @@ class Tracker(): best_results[group_id][1] = (lms, eye_state) sorted_results = sorted(best_results.values(), key=lambda x: x[0], reverse=True) - duration_model = 1000 * (time.time() - start_model) + duration_model = 1000 * (time.perf_counter() - start_model) results = [] detected = [] @@ -509,10 +492,9 @@ class Tracker(): break; if conf > self.threshold: detections += 1 - start_pnp = time.time() + start_pnp = time.perf_counter() success, quaternion, euler, pnp_error, pts_3d, lms = self.estimateDepth(lms, eye_state) - duration_pnp += 1000 * (time.time() - start_pnp) - results.append((conf, lms, success, pnp_error, quaternion, euler, self.rotation, self.translation, pts_3d, (eye_state[0][0], eye_state[1][0]))) + duration_pnp += 1000 * (time.perf_counter() - start_pnp) min_x = 10000 max_x = -1 min_y = 10000 @@ -529,6 +511,7 @@ class Tracker(): w = max_x - min_x y = max_y - min_y detected.append((min_y, min_x, max_y - min_y, max_x - min_x)) + results.append((conf, lms, success, pnp_error, quaternion, euler, self.rotation, self.translation, pts_3d, (eye_state[0][0], eye_state[1][0]), (min_y, min_x, max_y - min_y, max_x - min_x))) if len(detected) > 0: self.detected = len(detected) @@ -552,8 +535,10 @@ class Tracker(): faces.append((x,y,w,h)) self.faces = faces - duration = (time.time() - start) * 1000 + duration = (time.perf_counter() - start) * 1000 if not self.silent: print(f"Took {duration:.2f}ms (detect: {duration_fd:.2f}ms, crop: {duration_pp:.2f}, track: {duration_model:.2f}ms, 3D points: {duration_pnp:.2f}ms)") + + results = sorted(results, key=lambda x: x[10][1] * self.height + x[10][0]) return results