From a6a9ae0a88bcf37de04aa32a891e4025981e7065 Mon Sep 17 00:00:00 2001 From: Cowwan Date: Tue, 6 Feb 2024 23:37:57 -0500 Subject: [PATCH] removed the message queue and thread, it was a silly idea --- eyes.py | 10 ++++------ facedetection.py | 4 ---- facetracker.py | 36 +++++++----------------------------- tracker.py | 12 +++++------- vts.py | 5 ++--- webcam.py | 7 +++---- 6 files changed, 21 insertions(+), 53 deletions(-) diff --git a/eyes.py b/eyes.py index f0664c0..8db04fe 100644 --- a/eyes.py +++ b/eyes.py @@ -142,14 +142,13 @@ class EyeTracker(): self.rightEye = Eye(0) self.mean = np.float32(np.array([-2.1179, -2.0357, -1.8044])) self.std = np.float32(np.array([0.0171, 0.0175, 0.0174])) - self.faceFrame = None self.offset = None self.faceCenter = None self.faceRadius = None def get_eye_state(self, models, frame, lms): - lms = self.extract_face(frame, np.array(lms)[:,0:2][:,::-1]) + lms, faceFrame = self.extract_face(frame, np.array(lms)[:,0:2][:,::-1]) self.rightEye.offset = self.offset self.leftEye.offset = self.offset @@ -158,8 +157,8 @@ class EyeTracker(): self.leftEye.innerPoint = lms[45,0:2] self.leftEye.outerPoint = lms[42,0:2] - self.rightEye.prepare_eye(self.faceFrame) - self.leftEye.prepare_eye(self.faceFrame) + self.rightEye.prepare_eye(faceFrame) + self.leftEye.prepare_eye(faceFrame) if self.rightEye.image is None or self.leftEye.image is None: return [[1.,0.,0.,0.],[1.,0.,0.,0.]] #Early exit if one of the eyes doesn't have data @@ -182,5 +181,4 @@ class EyeTracker(): x2, y2 = clamp_to_im(self.faceCenter + self.faceRadius + 1, h, w) self.offset = np.array((x1, y1)) lms = (lms[:, 0:2] - self.offset).astype(int) - self.faceFrame = frame[y1:y2, x1:x2] - return lms + return lms, frame[y1:y2, x1:x2] diff --git a/facedetection.py b/facedetection.py index 32f40fc..db84b2a 100644 --- a/facedetection.py +++ b/facedetection.py @@ -1,10 +1,6 @@ import numpy as np import cv2 - - - - def detect_faces(frame, model, detection_threshold ): image = resizeImage(frame) diff --git a/facetracker.py b/facetracker.py index d23b65a..d3ea089 100644 --- a/facetracker.py +++ b/facetracker.py @@ -16,10 +16,6 @@ import vts import preview import cv2 -#overall, my changes seem to have memory usage up from about 140mb to 180mb -#but cpu usage seems to be down -#and it's more resilent to low performance scenarios - parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("-i", "--ip", help="Set IP address for sending tracking data", default="127.0.0.1") parser.add_argument("-p", "--port", type=int, help="Set port for sending tracking data", default=11573) @@ -41,14 +37,6 @@ parser.add_argument("-v", "--visualize", type=int, help="Set this to 1 to visual args = parser.parse_args() -os.environ["OMP_NUM_THREADS"] = str(args.landmark_detection_threads) - -def messaging(messageQueue): - while True: - message = messageQueue.get() - print(message) - - def visualize(frame, face, previewFrameQueue, face_center, face_radius): y1 = int(face_center[1] - face_radius[1]) @@ -103,17 +91,10 @@ if fps > 30: #creating queues frameQueue = multiprocessing.Queue(maxsize=frameQueueSize) faceQueue = multiprocessing.Queue(maxsize=1) -messageQueue = queue.Queue() packetQueue = queue.Queue() -#creating threads -#probably doesn't need it's own thread, but it can't hurt -messageThread = threading.Thread(target = messaging, args = [ messageQueue],) -messageThread.daemon = True -messageThread.start() - #I want to decouple the requests so they don't block anything else if they're slow -VTS = vts.VTS(target_ip, target_port, silent, height, width, packetQueue, messageQueue) +VTS = vts.VTS(target_ip, target_port, silent, height, width, packetQueue) packetSenderThread = threading.Thread(target = VTS.packetSender, args = [],) packetSenderThread.daemon = True packetSenderThread.start() @@ -122,7 +103,7 @@ packetSenderThread.start() #not a very active process, it's blocked by the main thread 90% of the time #but putting it in it's own process means there's a little bit less occupying the same core as the main thread #which helps when the system is under load -webcamProcess = multiprocessing.Process(target=webcam.startProcess,args = (frameQueue, faceQueue, fps, targetBrightness, width, height, mirror_input,messageQueue, )) +webcamProcess = multiprocessing.Process(target=webcam.startProcess,args = (frameQueue, faceQueue, fps, targetBrightness, width, height, mirror_input, )) webcamProcess.daemon = True webcamProcess.start() @@ -143,7 +124,7 @@ peak_camera_latency = 0.0 total_camera_latency = 0.0 failures = 0 -tracker = Tracker(width, height, messageQueue, featureType, threshold=args.threshold, silent=silent, model_type=args.model, detection_threshold=args.detection_threshold) +tracker = Tracker(width, height, featureType, threshold=args.threshold, silent=silent, model_type=args.model, detection_threshold=args.detection_threshold) #don't start until the webcam is ready while frameQueue.qsize() < 1: @@ -170,14 +151,13 @@ try: #there's no reason to ever wait on this process or give it much of a queue if previewFrameQueue.qsize() < 1: previewFrameQueue.put(frame) - try: face, face_center, face_radius = tracker.predict(frame) failures = 0 except Exception as e: if e.__class__ == KeyboardInterrupt: if not silent: - messageQueue.put("Quitting") + print("Quitting") break traceback.print_exc() failures += 1 @@ -186,14 +166,12 @@ try: if face is not None: packet = VTS.preparePacket(face) - #this can maybe cause weirdness in the gamma calculations - #but I don't want to let that build a queue if faceQueue.qsize() < 1: faceQueue.put([face_center, face_radius]) if visualizeFlag: visualize(frame, face, previewFrameQueue, face_center, face_radius) else: - messageQueue.put("Camera behind, skipping frame") + print("Camera behind, skipping frame") duration = time.perf_counter() - frame_start @@ -203,7 +181,7 @@ try: time.sleep(target_duration - duration) else: #all caps because if this is consistently happening it's an issue - messageQueue.put("CANNOT MAINTAIN FRAMERATE!") + print("CANNOT MAINTAIN FRAMERATE!") peak_time_between = max(peak_time_between, time.perf_counter() -frame_start) total_run_time += time.perf_counter() -frame_start @@ -212,7 +190,7 @@ try: if(packet is not None): packetQueue.put(packet) else: - messageQueue.put("No data sent to VTS") + print("No data sent to VTS") except KeyboardInterrupt: if not silent: diff --git a/tracker.py b/tracker.py index 6a698b5..bdbc357 100644 --- a/tracker.py +++ b/tracker.py @@ -21,8 +21,6 @@ def clamp_to_im(pt, w, h): #8 times per frame, but that only accounts for 0.005m y = h-1 return (int(x),int(y+1)) - - class Models(): def __init__(self, model_type=3): self.model_type = model_type @@ -50,7 +48,7 @@ class Models(): self.gazeTracker = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options, providers=providersList) class Tracker(): - def __init__(self, width, height, messageQueue, featureType, model_type=3, detection_threshold=0.6, threshold=0.6, silent=False): + def __init__(self, width, height, featureType, model_type=3, detection_threshold=0.6, threshold=0.6, silent=False): self.detection_threshold = detection_threshold self.EyeTracker = eyes.EyeTracker() @@ -66,7 +64,6 @@ class Tracker(): self.threshold = threshold self.face = None self.face_info = face.FaceInfo(featureType) - self.messageQueue = messageQueue #scales cropped frame before sending to landmarks def preprocess(self, im): @@ -103,10 +100,10 @@ class Tracker(): return (crop, crop_info, duration_pp ) def early_exit(self, reason, start): - self.messageQueue.put(reason) + print(reason) self.face = None duration = (time.perf_counter() - start) * 1000 - self.messageQueue.put(f"Took {duration:.2f}ms") + print(f"Took {duration:.2f}ms") return None, None, None def predict(self, frame): @@ -129,6 +126,7 @@ class Tracker(): if crop is None: return self.early_exit("No valid crops", start) start_model = time.perf_counter() + confidence, lms = landmarks.run(self.model, crop, crop_info) #Early exit if below confidence threshold if confidence < self.threshold: @@ -153,7 +151,7 @@ class Tracker(): self.face = [y1, x1, y2 - y1, x2 - x1] duration_pnp += 1000 * (time.perf_counter() - start_pnp) duration = (time.perf_counter() - start) * 1000 - self.messageQueue.put(f"Took {duration:.2f}ms (detect: {duration_fd:.2f}ms, crop: {duration_pp:.2f}ms, track: {duration_model:.2f}ms, 3D points: {duration_pnp:.2f}ms)") + print(f"Took {duration:.2f}ms (detect: {duration_fd:.2f}ms, crop: {duration_pp:.2f}ms, track: {duration_model:.2f}ms, 3D points: {duration_pnp:.2f}ms)") return face_info, self.EyeTracker.faceCenter, self.EyeTracker.faceRadius #Combined multiple failures into one catch all exit diff --git a/vts.py b/vts.py index aa0442f..a1944ff 100644 --- a/vts.py +++ b/vts.py @@ -3,7 +3,7 @@ import socket import struct class VTS(): - def __init__(self, target_ip, target_port, silent, height, width, packetQueue, messageQueue): + def __init__(self, target_ip, target_port, silent, height, width, packetQueue): self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.target_ip = target_ip @@ -12,7 +12,6 @@ class VTS(): self.features = ["eye_l", "eye_r", "eyebrow_steepness_l", "eyebrow_updown_l", "eyebrow_quirk_l", "eyebrow_steepness_r", "eyebrow_updown_r", "eyebrow_quirk_r", "mouth_corner_updown_l", "mouth_corner_inout_l", "mouth_corner_updown_r", "mouth_corner_inout_r", "mouth_open", "mouth_wide"] self.width = width self.height = height - self.messageQueue = messageQueue self.packetQueue = packetQueue def packetSender(self): @@ -68,5 +67,5 @@ class VTS(): try: self.sock.sendto(packet, (self.target_ip, self.target_port)) except: - self.messageQueue.put("Failed to send packet") + print("Failed to send packet") return diff --git a/webcam.py b/webcam.py index d56138b..71f36b7 100644 --- a/webcam.py +++ b/webcam.py @@ -6,12 +6,12 @@ import math #this is just here for conveience -def startProcess(frameQueue, faceQueue, fps, targetBrightness, width, height, mirrorInput, messageQueue): - webcam = Webcam(frameQueue, faceQueue, fps, targetBrightness, width, height, mirrorInput, messageQueue) +def startProcess(frameQueue, faceQueue, fps, targetBrightness, width, height, mirrorInput): + webcam = Webcam(frameQueue, faceQueue, fps, targetBrightness, width, height, mirrorInput) webcam.start() class Webcam(): - def __init__(self, frameQueue, faceQueue, fps, targetBrightness, width, height, mirrorInput, messageQueue): + def __init__(self, frameQueue, faceQueue, fps, targetBrightness, width, height, mirrorInput): self.cap = cv2.VideoCapture(0) #tbh, these settings will make things way slower and frametimes will suffer #but they're the only way frames bigger than 480p will work @@ -31,7 +31,6 @@ class Webcam(): self.cameraLatency = 0 self.brightnessFrame = None #a copy of the brightness channel for gamma calculations self.targetBrightness = targetBrightness #the target average brightness of the face, used in gamma calculations - self.messageQueue = messageQueue #outgoing queue, unused, but I wanted it here in case I wanted it self.width = width #unused, but it seemed useful to have around self.height = height #unused, but it seemed useful to have around self.frame = None