diff --git a/facetracker.py b/facetracker.py index 259f382..e161e07 100644 --- a/facetracker.py +++ b/facetracker.py @@ -1,8 +1,4 @@ -#import copy import os -#limits the threads used by numpy to 1 -#setting this and the landmarks model to use 1 means the cpu usage is pretty low - os.environ["OMP_NUM_THREADS"] = str(1) import argparse import traceback @@ -14,7 +10,6 @@ import multiprocessing import webcam import vts import preview -import cv2 parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("-i", "--ip", help="Set IP address for sending tracking data", default="127.0.0.1") @@ -31,33 +26,12 @@ parser.add_argument("--model", type=int, help="This can be used to select the tr parser.add_argument("--preview", type=int, help="Preview the frames sent to the tracker", default=0) parser.add_argument("--feature-type", type=int, help="Sets which version of feature extraction is used. 0 is my new version that works well for me and allows for some customization, 1 is EmilianaVT's version aka, normal OpenSeeFace operation", default=0, choices=[0, 1]) parser.add_argument("--numpy-threads", type=int, help="Numer of threads Numpy can use, doesn't seem to effect much", default=1) -parser.add_argument("--landmark-detection-threads", type=int, help="Numer of threads used for landmark detection. Default is 1 (~15ms per frame on my computer), 2 gets slightly faster frames (~10ms on my computer), more than 2 doesn't seem to help much", default=1) +parser.add_argument("-T","--threads", type=int, help="Numer of threads used for landmark detection. Default is 1 (~15ms per frame on my computer), 2 gets slightly faster frames (~10ms on my computer), more than 2 doesn't seem to help much", default=1) parser.add_argument("-v", "--visualize", type=int, help="Set this to 1 to visualize the tracking", default=0) args = parser.parse_args() -def visualize(frame, face, previewFrameQueue, face_center, face_radius): - - y1 = int(face_center[1] - face_radius[1]) - y2 = int(face_center[1] + face_radius[1]) - x1 = int(face_center[0] - face_radius[0]) - x2 = int(face_center[0] + face_radius[0]) - bbox = (y1, x1, y2 - y1, x2 - x1) - frame = cv2.putText(frame, str(face.id), (int(bbox[1]), int(bbox[0])), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (255,0,255)) - frame = cv2.putText(frame, f"{face.conf:.4f}", (int(bbox[1] + 18), int(bbox[0] - 6)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,0,255)) - for pt_num, (x,y,c) in enumerate(face.lms): - x = int(x + 0.5) - y = int(y + 0.5) - frame = cv2.putText(frame, str(pt_num), (int(y), int(x)), cv2.FONT_HERSHEY_SIMPLEX, 0.25, (255,255,0)) - color = (0, 255, 0) - if pt_num >= 66: - color = (255, 255, 0) - if not (x < 0 or y < 0 or x >= height or y >= width): - cv2.circle(frame, (y, x), 1, color, -1) - previewFrameQueue.put(frame) - - #processing args #For Vtube Studio: ip 127.0.0.1 port 11573 target_ip = args.ip @@ -69,6 +43,7 @@ previewFlag = (args.preview == 1) mirror_input = args.mirror_input featureType = args.feature_type visualizeFlag = (args.visualize == 1) +threads = args.threads target_duration = 1. / fps targetBrightness = 0.55 @@ -115,8 +90,10 @@ total_camera_latency = 0.0 sleepTimer = 0.0 failures = 0 +peakTotalLatency = 0.0 +totalTotalLatency = 0.0 -tracker = Tracker(width, height, featureType, threshold=args.threshold, silent=silent, model_type=args.model, detection_threshold=args.detection_threshold) +tracker = Tracker(width, height, featureType, threads, threshold=args.threshold, silent=silent, model_type=args.model, detection_threshold=args.detection_threshold) #don't start until the webcam is ready, then give it a little more time to fill it's buffer while frameQueue.qsize() < 1: @@ -130,49 +107,35 @@ try: packet = None frame_start = time.perf_counter() frame_count += 1 - if True: - frame, camera_latency, totalFrameLatency = frameQueue.get() - frame_get = time.perf_counter() - #If I don't wait a few frames to start tracking I get wild peak frame times, like 500ms - if frame_count > 5: - peak_camera_latency = max(camera_latency, peak_camera_latency) - total_camera_latency+= camera_latency + frame, camera_latency, totalFrameLatency = frameQueue.get() + frame_get = time.perf_counter() + #If I don't wait a few frames to start tracking I get wild peak frame times, like 500ms + if frame_count > 5: + peak_camera_latency = max(camera_latency, peak_camera_latency) + total_camera_latency+= camera_latency - if previewFlag: - #there's no reason to ever wait on this process or give it much of a queue - if previewFrameQueue.qsize() < 1: - previewFrameQueue.put(frame) - - try: - face, face_center, face_radius = tracker.predict(frame) - failures = 0 - except Exception as e: - if e.__class__ == KeyboardInterrupt: - if not silent: - print("Quitting") - break - traceback.print_exc() - failures += 1 - if failures > 30: - break + if previewFlag: + #there's no reason to ever wait on this process or give it much of a queue + if previewFrameQueue.qsize() < 1: + previewFrameQueue.put(frame) + face, face_center, face_radius = tracker.predict(frame) if face is not None: packet = VTS.preparePacket(face) if faceQueue.qsize() < 1: faceQueue.put([face_center, face_radius]) if visualizeFlag: - visualize(frame, face, previewFrameQueue, face_center, face_radius) + preview.visualize(frame, face, previewFrameQueue, face_center, face_radius) duration = time.perf_counter() - frame_get - total_active_time += duration peak_frame_time = max(peak_frame_time, duration) sleepTimer = max(time.perf_counter() - frame_get, sleepTimer) time.sleep(max(peak_frame_time - duration, 0)) - sleepTimer -= 0.0001 + sleepTimer -= 0.0005 peak_time_between = max(peak_time_between, time.perf_counter() -frame_start) total_run_time += time.perf_counter() -frame_start @@ -182,7 +145,9 @@ try: packetQueue.put(packet) else: print("No data sent to VTS") - print(time.perf_counter() - totalFrameLatency) + if frame_count > 5: + peakTotalLatency = max(peakTotalLatency, (time.perf_counter() - totalFrameLatency)) + totalTotalLatency += time.perf_counter() - totalFrameLatency except KeyboardInterrupt: if not silent: @@ -190,6 +155,9 @@ except KeyboardInterrupt: #printing statistics on close #it makes identifying problems easier + +print(f"Peak latency: {(peakTotalLatency*1000):.3f}ms") +print(f"Average latency: {(totalTotalLatency*1000/(frame_count-5)):.3f}ms") #480p generally results in camera latencies between 0.5ms and 2.0ms print(f"Peak camera latency: {(peak_camera_latency*1000):.3f}ms") print(f"Average camera latency: {(total_camera_latency*1000/(frame_count-5)):.3f}ms") diff --git a/facetracking.sh b/facetracking.sh index 810a561..d794811 100755 --- a/facetracking.sh +++ b/facetracking.sh @@ -16,5 +16,5 @@ python3 -m venv .venv source .venv/bin/activate #the openseeface command -python3 facetracker.py --fps 30 --ip 127.0.0.1 --port 11573 --model 3 -W 640 -H 480 #-W 1920 -H 1080 #-W 640 -H 480 #--preview 1 +python3 facetracker.py --fps 30 --ip 127.0.0.1 --port 11573 --model 3 -W 640 -H 480 #-W 1920 -H 1080 #-W 640 -H 480 #--preview 1 diff --git a/preview.py b/preview.py index 9c9da64..eb9e401 100644 --- a/preview.py +++ b/preview.py @@ -19,3 +19,22 @@ class Preview(): cv2.imshow("test",cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)) cv2.waitKey(1) +def visualize(frame, face, previewFrameQueue, face_center, face_radius): + + y1 = int(face_center[1] - face_radius[1]) + y2 = int(face_center[1] + face_radius[1]) + x1 = int(face_center[0] - face_radius[0]) + x2 = int(face_center[0] + face_radius[0]) + bbox = (y1, x1, y2 - y1, x2 - x1) + frame = cv2.putText(frame, str(face.id), (int(bbox[1]), int(bbox[0])), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (255,0,255)) + frame = cv2.putText(frame, f"{face.conf:.4f}", (int(bbox[1] + 18), int(bbox[0] - 6)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,0,255)) + for pt_num, (x,y,c) in enumerate(face.lms): + x = int(x + 0.5) + y = int(y + 0.5) + frame = cv2.putText(frame, str(pt_num), (int(y), int(x)), cv2.FONT_HERSHEY_SIMPLEX, 0.25, (255,255,0)) + color = (0, 255, 0) + if pt_num >= 66: + color = (255, 255, 0) + if not (x < 0 or y < 0 or x >= height or y >= width): + cv2.circle(frame, (y, x), 1, color, -1) + previewFrameQueue.put(frame) diff --git a/tracker.py b/tracker.py index bdbc357..b4de59d 100644 --- a/tracker.py +++ b/tracker.py @@ -22,7 +22,7 @@ def clamp_to_im(pt, w, h): #8 times per frame, but that only accounts for 0.005m return (int(x),int(y+1)) class Models(): - def __init__(self, model_type=3): + def __init__(self, threads, model_type=3): self.model_type = model_type self.models = [ "lm_model0_opt.onnx", @@ -36,7 +36,7 @@ class Models(): providersList = onnxruntime.capi._pybind_state.get_available_providers() options = onnxruntime.SessionOptions() options.inter_op_num_threads = 1 - options.intra_op_num_threads = 1 + options.intra_op_num_threads = threads options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL options.log_severity_level = 3 @@ -48,11 +48,11 @@ class Models(): self.gazeTracker = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options, providers=providersList) class Tracker(): - def __init__(self, width, height, featureType, model_type=3, detection_threshold=0.6, threshold=0.6, silent=False): + def __init__(self, width, height, featureType, threads, model_type=3, detection_threshold=0.6, threshold=0.6, silent=False): self.detection_threshold = detection_threshold self.EyeTracker = eyes.EyeTracker() - self.model = Models(model_type = model_type) + self.model = Models(threads, model_type = model_type) # Image normalization constants self.mean = np.float32(np.array([-2.1179, -2.0357, -1.8044])) diff --git a/webcam.py b/webcam.py index cfbbcb5..1b66e11 100644 --- a/webcam.py +++ b/webcam.py @@ -40,19 +40,19 @@ class Webcam(): self.width = width #unused, but it seemed useful to have around self.height = height #unused, but it seemed useful to have around self.ret = 0 + self.frame = None def start(self): while self.cap.isOpened(): frameStart= time.perf_counter() totalFrameLatency = time.perf_counter() - frame = self.getFrame() + self.getFrame() self.cameraLatency = time.perf_counter() - frameStart if self.ret: - frame = self.applyGamma(frame) + self.applyGamma() if self.mirror: - frame = cv2.flip(frame, 1) - #this line is where this process sits 90% of the time - self.frameQueue.put([frame, self.cameraLatency, totalFrameLatency]) + self.frame = cv2.flip(self.frame, 1) + self.frameQueue.put([self.frame, self.cameraLatency, totalFrameLatency]) if self.faceQueue.qsize() > 0: self.updateGamma() sleepTime = self.targetFrameTime - (time.perf_counter() - frameStart) @@ -67,15 +67,15 @@ class Webcam(): #keeping track of how long it takes to get frames from the webcam #because that one line is the cause of 90% of late frames cameraStart = time.perf_counter() - self.ret, frame = self.cap.read() + self.ret, self.frame = self.cap.read() self.cameraLatency = time.perf_counter() - cameraStart - return(frame) + return #Applies a gamma curve to the frame #Uses a lookup table because that's way faster than actually calculating every pixel #Immensely improves tracking in low light situations - def applyGamma(self, frame): - img_yuv = cv2.cvtColor(frame, cv2.COLOR_BGR2YUV) + def applyGamma(self): + img_yuv = cv2.cvtColor(self.frame, cv2.COLOR_BGR2YUV) #saving a copy of the brightness channel so I can use it to adjust the gamma based on where the face is in that frame self.brightnessFrame = img_yuv[:,:,0].copy() #building a lookup table on the fly @@ -84,8 +84,8 @@ class Webcam(): lookupTable = np.power(loopupTable, self.gamma)*255 img_yuv[:,:,0] = lookupTable[img_yuv[:,:,0]] #I convert the image to RBG here because it was getting repeatedly converted in the face tracking - frame = cv2.cvtColor(img_yuv, cv2.COLOR_YUV2RGB) - return frame + self.frame = cv2.cvtColor(img_yuv, cv2.COLOR_YUV2RGB) + #calculate the ideal gamma based on the brightness of the user's face