From f288689960bdb95ee77406521766fe3e9c6d5a9d Mon Sep 17 00:00:00 2001 From: Cowwan Date: Fri, 9 Feb 2024 20:39:51 -0500 Subject: [PATCH] stuff --- .stats.kate-swp | Bin 0 -> 2419 bytes eyes.py | 11 ++-- face.py | 147 ++++++++++++++++++++++++----------------------- facedetection.py | 39 +++++++------ facetracker.py | 30 +++++----- image.py | 0 landmarks.py | 1 + preview.py | 1 + tracker.py | 54 +++++++++-------- vts.py | 3 +- webcam.py | 35 ++++++++--- 11 files changed, 175 insertions(+), 146 deletions(-) create mode 100644 .stats.kate-swp create mode 100644 image.py diff --git a/.stats.kate-swp b/.stats.kate-swp new file mode 100644 index 0000000000000000000000000000000000000000..bfd3cdca26181591e066150b5ee6a903af61cc75 GIT binary patch literal 2419 zcmb7`NpBQK6vxM`F$w#697O_~16I9SUx~Ak5I3BNvjGb%VL-MKBDWm41Z=j1eTOI) z#24TrMB)_X#<$?aiT8iqGu70x9h+85AWXd+wtFje)sY4)&D>J^XAaMGe_mlA9CUtJ2O4hW6-ljuYq1DdTpiG zwZA_hpLpK-=@|^K$=PeD>ln&0@ypr8x!Kc8ebZmObbfh$VXkkcj3KfvcnWNi4>=eI z9Lz-Xq!qAb{`I29`bBSmo-BGJ^!cKHfWB1pICLkUP0$z1{U4#P6#WzQ!6uY8DwIy6 zlFx4?WpZG$*;8VllkrFjj3jlu9U`*vW+L3tK&jl8uX+>s?XZ3jgO+XnRm2TkR&iOu zZlE_yZ+bNVb_Eh=3HukroweS~;*1|6pt%dJ2M)T?1*TQ$6uZ8azO$*o#@^bjP?`p=JJo5x0G}5tB=qT+H4&n~c?-Y0JObaQf24DHuy85gX#{ zJPjKTcvVguL$}xp_9;20WOL>ihyjC{0DMDE97EVuaFl4pZYFz|DJu*{;hTzfU|+$W zD4S?qvYwr@m>G=1w_5nXVC9Aij=@W~GI6#sC=28OqQ2d-!>D7$21|~dB_fuQu6i&! z@tqn@H5{`Pi!w>IZOn)ojKY7`FxPPKnX`3_^6TTXb@OQnk^g5Il=Hto87jrVMBj^t};!#9(ZsLD?~r4=y2WKuX_l>0!00c375; zC^;_oiM+`gHyDKhW+tdT0cqTiMy9ST z#%z-}F>(%8yoRK~=+uu}xJ+Gnj1#jlK5#H8QnR1|>C{hJx|FOk8FNTtl3?T+)VLdr zPW^NQu1qFtDquDG6eu{|4ZtY;tc6S5mC0mc*+N3{Vv;1aN7jHe?&mdaE1R(_NA?`0 z;9fQ&r2#4ZVuY@2ChH=JK{!VrqA|Lt0V(}*gsyCsP4G!7ZBmfuH2ngk^s5oN@)^e* zZMK|T7T87P4M<+^7`_JI4F7J1SU4Bt+%D(fPXY7}==M(m^iJsZ-2=S~y8TlCy&L*u W^R9v31KqxRp!Y&|_ 0.15 - outputs[0] = outputs[0] * mask.astype(int) * mask2.astype(int) - faceLocation = np.argmax(outputs[0].flatten()) - x = faceLocation % 56 - y = faceLocation // 56 - if outputs[0, y, x] < detection_threshold: - return None - r = outputs[1, y, x] * 112. - results= (((x * 4) - r, (y * 4) - r, r*2,r*2)) - results = np.array(results).astype(np.float32) - results[[0,2]] *= frame.shape[1] / 224. - results[[1,3]] *= frame.shape[0] / 224. + outputs, maxpool = model.faceDetection.run([], {'input': image}) + outputs = outputs[0] + maxpool = maxpool[0] + mask = outputs[0] == maxpool[0] + mask2 = outputs[1] > 0.15 + outputs[0] = outputs[0] * mask.astype(int) * mask2.astype(int) + faceLocation = np.argmax(outputs[0].flatten()) + x = faceLocation % 56 + y = faceLocation // 56 + if outputs[0, y, x] < detection_threshold: + return None + r = outputs[1, y, x] * 112. + results= (((x * 4) - r, (y * 4) - r, r*2,r*2)) + results = np.array(results).astype(np.float32) + results[[0,2]] *= frame.shape[1] / 224. + results[[1,3]] *= frame.shape[0] / 224. - return results + return results def resizeImage(frame): mean = np.float32(np.array([-2.1179, -2.0357, -1.8044])) std = np.float32(np.array([0.0171, 0.0175, 0.0174])) targetimageSize = [224, 224] - image = cv2.resize(frame, targetimageSize, interpolation=cv2.INTER_LINEAR) * std + mean + image = cv2.resize(frame, targetimageSize, interpolation=cv2.INTER_CUBIC) * std + mean image = np.expand_dims(image, 0) image = np.transpose(image, (0,3,1,2)) return image diff --git a/facetracker.py b/facetracker.py index e161e07..223569c 100644 --- a/facetracker.py +++ b/facetracker.py @@ -26,14 +26,13 @@ parser.add_argument("--model", type=int, help="This can be used to select the tr parser.add_argument("--preview", type=int, help="Preview the frames sent to the tracker", default=0) parser.add_argument("--feature-type", type=int, help="Sets which version of feature extraction is used. 0 is my new version that works well for me and allows for some customization, 1 is EmilianaVT's version aka, normal OpenSeeFace operation", default=0, choices=[0, 1]) parser.add_argument("--numpy-threads", type=int, help="Numer of threads Numpy can use, doesn't seem to effect much", default=1) -parser.add_argument("-T","--threads", type=int, help="Numer of threads used for landmark detection. Default is 1 (~15ms per frame on my computer), 2 gets slightly faster frames (~10ms on my computer), more than 2 doesn't seem to help much", default=1) +parser.add_argument("-T","--threads", type=int, help="Numer of threads used for landmark detection. Default is 1 (~15ms per frame on my computer), 2 gets slightly faster frames (~10ms on my computer), more than 2 doesn't seem to help much", default=2) parser.add_argument("-v", "--visualize", type=int, help="Set this to 1 to visualize the tracking", default=0) args = parser.parse_args() #processing args -#For Vtube Studio: ip 127.0.0.1 port 11573 target_ip = args.ip target_port = args.port fps = args.fps @@ -53,7 +52,8 @@ silent = (args.silent == 1) if height > 480: frameQueueSize = 2 else: - frameQueueSize = 1 + frameQueueSize = 2 + frameQueue = multiprocessing.Queue(maxsize=frameQueueSize) faceQueue = multiprocessing.Queue(maxsize=1) packetQueue = queue.Queue() @@ -65,9 +65,7 @@ packetSenderThread.daemon = True packetSenderThread.start() #creating processes -#not a very active process, it's blocked by the main thread 90% of the time -#but putting it in it's own process means there's a little bit less occupying the same core as the main thread -#which helps when the system is under load + webcamProcess = multiprocessing.Process(target=webcam.startProcess,args = (frameQueue, faceQueue, fps, targetBrightness, width, height, mirror_input, )) webcamProcess.daemon = True webcamProcess.start() @@ -119,23 +117,23 @@ try: if previewFrameQueue.qsize() < 1: previewFrameQueue.put(frame) - face, face_center, face_radius = tracker.predict(frame) + faceInfo, face = tracker.predict(frame) - if face is not None: - packet = VTS.preparePacket(face) + if faceInfo is not None: + packet = VTS.preparePacket(faceInfo) if faceQueue.qsize() < 1: - faceQueue.put([face_center, face_radius]) + faceQueue.put(face) if visualizeFlag: - preview.visualize(frame, face, previewFrameQueue, face_center, face_radius) + preview.visualize(frame, faceInfo, previewFrameQueue, face_center, face_radius) duration = time.perf_counter() - frame_get total_active_time += duration peak_frame_time = max(peak_frame_time, duration) - - - sleepTimer = max(time.perf_counter() - frame_get, sleepTimer) - time.sleep(max(peak_frame_time - duration, 0)) - sleepTimer -= 0.0005 + if duration < target_duration: + time.sleep(target_duration - duration) + else: + #all caps because if this is consistently happening it's an issue + messageQueue.put("CANNOT MAINTAIN FRAMERATE!") peak_time_between = max(peak_time_between, time.perf_counter() -frame_start) total_run_time += time.perf_counter() -frame_start diff --git a/image.py b/image.py new file mode 100644 index 0000000..e69de29 diff --git a/landmarks.py b/landmarks.py index 882fe20..0ce3c72 100644 --- a/landmarks.py +++ b/landmarks.py @@ -1,6 +1,7 @@ import numpy as np import math import cv2 +cv2.setNumThreads(6) def matrix_to_quaternion(m): t = 0.0 diff --git a/preview.py b/preview.py index eb9e401..f55b866 100644 --- a/preview.py +++ b/preview.py @@ -1,5 +1,6 @@ import cv2 import multiprocessing +cv2.setNumThreads(6) #this is it's own file just for ceanlieness diff --git a/tracker.py b/tracker.py index b4de59d..9dbb8f3 100644 --- a/tracker.py +++ b/tracker.py @@ -2,6 +2,7 @@ import os import numpy as np import onnxruntime import cv2 +cv2.setNumThreads(6) import time import facedetection import eyes @@ -9,27 +10,34 @@ import landmarks import face + #this file is so much smaller than it used to be, lol #I moved most of the functionality into separate files so they're easier to work with -def clamp_to_im(pt, w, h): #8 times per frame, but that only accounts for 0.005ms - x=max(pt[0],0) - y=max(pt[1],0) +def clamp_to_im(pt, w, h): + x = pt[0] + y = pt[1] + if x < 0: + x = 0 + if y < 0: + y = 0 if x >= w: x = w-1 if y >= h: y = h-1 - return (int(x),int(y+1)) + return (int(x), int(y+1)) class Models(): + + models = [ + "lm_model0_opt.onnx", + "lm_model1_opt.onnx", + "lm_model2_opt.onnx", + "lm_model3_opt.onnx", + "lm_model4_opt.onnx"] + def __init__(self, threads, model_type=3): self.model_type = model_type - self.models = [ - "lm_model0_opt.onnx", - "lm_model1_opt.onnx", - "lm_model2_opt.onnx", - "lm_model3_opt.onnx", - "lm_model4_opt.onnx"] model = self.models[self.model_type] model_base_path = os.path.join(os.path.dirname(__file__), os.path.join("models")) @@ -48,6 +56,9 @@ class Models(): self.gazeTracker = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options, providers=providersList) class Tracker(): + targetimageSize = [224, 224] + std = np.float32(np.array([0.0171, 0.0175, 0.0174])) + mean = np.float32(np.array([-2.1179, -2.0357, -1.8044])) def __init__(self, width, height, featureType, threads, model_type=3, detection_threshold=0.6, threshold=0.6, silent=False): self.detection_threshold = detection_threshold @@ -55,9 +66,6 @@ class Tracker(): self.model = Models(threads, model_type = model_type) # Image normalization constants - self.mean = np.float32(np.array([-2.1179, -2.0357, -1.8044])) - self.std = np.float32(np.array([0.0171, 0.0175, 0.0174])) - self.targetimageSize = [224, 224] self.width = width self.height = height @@ -75,12 +83,10 @@ class Tracker(): duration_pp = 0.0 x,y,w,h = self.face - wint = int(w * 0.1) - hint = int(h * 0.125) - crop_x1 = x - wint - crop_y1 = y - hint - crop_x2 = x + w + wint - crop_y2 = y + h + hint + crop_x1 = x - (w * 0.1) + crop_y1 = y - (h * 0.125) + crop_x2 = x + w + (w * 0.1) + crop_y2 = y + h + (h * 0.125) crop_x1, crop_y1 = clamp_to_im((crop_x1, crop_y1), self.width, self.height) crop_x2, crop_y2 = clamp_to_im((crop_x2, crop_y2), self.width, self.height) @@ -104,7 +110,7 @@ class Tracker(): self.face = None duration = (time.perf_counter() - start) * 1000 print(f"Took {duration:.2f}ms") - return None, None, None + return None, None def predict(self, frame): start = time.perf_counter() @@ -146,13 +152,13 @@ class Tracker(): if face_info.success: face_info.adjust_3d() lms = face_info.lms[:, 0:2] - x1, y1 = lms[0:66].min(0) - x2, y2 = lms[0:66].max(0) - self.face = [y1, x1, y2 - y1, x2 - x1] + y1, x1 = lms[0:66].min(0) + y2, x2 = lms[0:66].max(0) + self.face = [x1, y1, x2 - x1, y2 - y1] duration_pnp += 1000 * (time.perf_counter() - start_pnp) duration = (time.perf_counter() - start) * 1000 print(f"Took {duration:.2f}ms (detect: {duration_fd:.2f}ms, crop: {duration_pp:.2f}ms, track: {duration_model:.2f}ms, 3D points: {duration_pnp:.2f}ms)") - return face_info, self.EyeTracker.faceCenter, self.EyeTracker.faceRadius + return face_info, self.face #Combined multiple failures into one catch all exit return self.early_exit("Face info not valid", start) diff --git a/vts.py b/vts.py index a1944ff..8dfed2c 100644 --- a/vts.py +++ b/vts.py @@ -3,13 +3,14 @@ import socket import struct class VTS(): + + features = ["eye_l", "eye_r", "eyebrow_steepness_l", "eyebrow_updown_l", "eyebrow_quirk_l", "eyebrow_steepness_r", "eyebrow_updown_r", "eyebrow_quirk_r", "mouth_corner_updown_l", "mouth_corner_inout_l", "mouth_corner_updown_r", "mouth_corner_inout_r", "mouth_open", "mouth_wide"] def __init__(self, target_ip, target_port, silent, height, width, packetQueue): self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.target_ip = target_ip self.target_port = target_port self.silent = silent - self.features = ["eye_l", "eye_r", "eyebrow_steepness_l", "eyebrow_updown_l", "eyebrow_quirk_l", "eyebrow_steepness_r", "eyebrow_updown_r", "eyebrow_quirk_r", "mouth_corner_updown_l", "mouth_corner_inout_l", "mouth_corner_updown_r", "mouth_corner_inout_r", "mouth_open", "mouth_wide"] self.width = width self.height = height self.packetQueue = packetQueue diff --git a/webcam.py b/webcam.py index 1b66e11..538284b 100644 --- a/webcam.py +++ b/webcam.py @@ -3,11 +3,24 @@ import os os.environ["OMP_NUM_THREADS"] = str(1) import dshowcapture import cv2 +cv2.setNumThreads(6) import multiprocessing import time import numpy as np import math +def clamp_to_im(pt, w, h): + x = pt[0] + y = pt[1] + if x < 0: + x = 0 + if y < 0: + y = 0 + if x >= w: + x = w-1 + if y >= h: + y = h-1 + return (int(x), int(y+1)) #this is just here for conveience def startProcess(frameQueue, faceQueue, fps, targetBrightness, width, height, mirrorInput): @@ -43,6 +56,7 @@ class Webcam(): self.frame = None def start(self): + while self.cap.isOpened(): frameStart= time.perf_counter() totalFrameLatency = time.perf_counter() @@ -91,17 +105,20 @@ class Webcam(): #calculate the ideal gamma based on the brightness of the user's face #kind of winging it with the math, but it's a lot better than calculating based on the whole frame def updateGamma(self): - face_center, face_radius = self.faceQueue.get() + face = self.faceQueue.get() - y1 = int(face_center[1] - face_radius[1]) - y2 = int(face_center[1] + face_radius[1]) - x1 = int(face_center[0] - face_radius[0]) - x2 = int(face_center[0] + face_radius[0]) + x,y,w,h = face + crop_x1 = x + crop_y1 = y + crop_x2 = x + w + crop_y2 = y + h - #I was doing random sampling of the image brightness - #and that's probably brighter for the whole frame - #but for just the face this is pretty fast - averageBrightness = np.mean(self.brightnessFrame[y1:y2,x1:x2])/256 + crop_x1, crop_y1 = clamp_to_im((crop_x1, crop_y1), self.width, self.height) + crop_x2, crop_y2 = clamp_to_im((crop_x2, crop_y2), self.width, self.height) + + croppedFace = self.brightnessFrame[crop_y1:crop_y2, crop_x1:crop_x2] + + averageBrightness = np.mean(croppedFace)/256 self.gamma = math.log(self.targetBrightness, averageBrightness) return