diff --git a/.stats.kate-swp b/.stats.kate-swp new file mode 100644 index 0000000..bfd3cdc Binary files /dev/null and b/.stats.kate-swp differ diff --git a/eyes.py b/eyes.py index 71d25e2..05a6bcb 100644 --- a/eyes.py +++ b/eyes.py @@ -1,6 +1,7 @@ import numpy as np import math import cv2 +cv2.setNumThreads(6) def clamp_to_im(pt, w, h): x = pt[0] @@ -33,14 +34,14 @@ def rotate_image(image, a, center): #twice per frame, 0.2ms - 0.25ms each, impro return image class Eye(): + mean = np.float32(np.array([-2.1179, -2.0357, -1.8044])) + std = np.float32(np.array([0.0171, 0.0175, 0.0174])) def __init__(self,index): self.eye_tracking_frames = 0 self.index = index self.state = [1.,0.,0.,0.] self.image = None self.info = None - self.mean = np.float32(np.array([-2.1179, -2.0357, -1.8044])) - self.std = np.float32(np.array([0.0171, 0.0175, 0.0174])) self.results = None self.condifence = 0 self.standardDeviation = 0. @@ -144,12 +145,13 @@ class Eye(): self.standardDeviation = math.sqrt(self.averageEyeConfidenceVariance) class EyeTracker(): + np.float32(np.array([-2.1179, -2.0357, -1.8044])) + std = np.float32(np.array([0.0171, 0.0175, 0.0174])) def __init__(self): self.leftEye = Eye(1) self.rightEye = Eye(0) - self.mean = np.float32(np.array([-2.1179, -2.0357, -1.8044])) - self.std = np.float32(np.array([0.0171, 0.0175, 0.0174])) self.offset = None + self.face = None self.faceCenter = None self.faceRadius = None @@ -187,5 +189,6 @@ class EyeTracker(): x1, y1 = clamp_to_im(self.faceCenter - self.faceRadius , h, w) x2, y2 = clamp_to_im(self.faceCenter + self.faceRadius + 1, h, w) self.offset = np.array((x1, y1)) + self.face = [x1, y1, x2 - x1, y2 - y1] lms = (lms[:, 0:2] - self.offset).astype(int) return lms, frame[y1:y2, x1:x2] diff --git a/face.py b/face.py index ffafc8c..6b5bc32 100644 --- a/face.py +++ b/face.py @@ -4,81 +4,82 @@ import featureExtractor import emilianaFeatureExtractor class FaceInfo(): + face_3d = np.array([ + [ 0.4551769692672 , 0.300895790030204, -0.764429433974752], + [ 0.448998827123556, 0.166995837790733, -0.765143004071253], + [ 0.437431554952677, 0.022655479179981, -0.739267175112735], + [ 0.415033422928434, -0.088941454648772, -0.747947437846473], + [ 0.389123587370091, -0.232380029794684, -0.704788385327458], + [ 0.334630113904382, -0.361265387599081, -0.615587579236862], + [ 0.263725112132858, -0.460009725616771, -0.491479221041573], + [ 0.16241621322721 , -0.558037146073869, -0.339445180872282], + [ 0. , -0.621079019321682, -0.287294770748887], + [-0.16241621322721 , -0.558037146073869, -0.339445180872282], + [-0.263725112132858, -0.460009725616771, -0.491479221041573], + [-0.334630113904382, -0.361265387599081, -0.615587579236862], + [-0.389123587370091, -0.232380029794684, -0.704788385327458], + [-0.415033422928434, -0.088941454648772, -0.747947437846473], + [-0.437431554952677, 0.022655479179981, -0.739267175112735], + [-0.448998827123556, 0.166995837790733, -0.765143004071253], + [-0.4551769692672 , 0.300895790030204, -0.764429433974752], + [ 0.385529968662985, 0.402800553948697, -0.310031082540741], + [ 0.322196658344302, 0.464439136821772, -0.250558059367669], + [ 0.25409760441282 , 0.46420381416882 , -0.208177722146526], + [ 0.186875436782135, 0.44706071961879 , -0.145299823706503], + [ 0.120880983543622, 0.423566314072968, -0.110757158774771], + [-0.120880983543622, 0.423566314072968, -0.110757158774771], + [-0.186875436782135, 0.44706071961879 , -0.145299823706503], + [-0.25409760441282 , 0.46420381416882 , -0.208177722146526], + [-0.322196658344302, 0.464439136821772, -0.250558059367669], + [-0.385529968662985, 0.402800553948697, -0.310031082540741], + [ 0. , 0.293332603215811, -0.137582088779393], + [ 0. , 0.194828701837823, -0.069158109325951], + [ 0. , 0.103844017393155, -0.009151819844964], + [ 0. , 0. , 0. ], + [ 0.080626352317973, -0.041276068128093, -0.134161035564826], + [ 0.046439347377934, -0.057675223874769, -0.102990627164664], + [ 0. , -0.068753126205604, -0.090545348482397], + [-0.046439347377934, -0.057675223874769, -0.102990627164664], + [-0.080626352317973, -0.041276068128093, -0.134161035564826], + [ 0.315905195966084, 0.298337502555443, -0.285107407636464], + [ 0.275252345439353, 0.312721904921771, -0.244558251170671], + [ 0.176394511553111, 0.311907184376107, -0.219205360345231], + [ 0.131229723798772, 0.284447361805627, -0.234239149487417], + [ 0.184124948330084, 0.260179585304867, -0.226590776513707], + [ 0.279433549294448, 0.267363071770222, -0.248441437111633], + [-0.131229723798772, 0.284447361805627, -0.234239149487417], + [-0.176394511553111, 0.311907184376107, -0.219205360345231], + [-0.275252345439353, 0.312721904921771, -0.244558251170671], + [-0.315905195966084, 0.298337502555443, -0.285107407636464], + [-0.279433549294448, 0.267363071770222, -0.248441437111633], + [-0.184124948330084, 0.260179585304867, -0.226590776513707], + [ 0.121155252430729, -0.208988660580347, -0.160606287940521], + [ 0.041356305910044, -0.194484199722098, -0.096159882202821], + [ 0. , -0.205180167345702, -0.083299217789729], + [-0.041356305910044, -0.194484199722098, -0.096159882202821], + [-0.121155252430729, -0.208988660580347, -0.160606287940521], + [-0.132325402795928, -0.290857984604968, -0.187067868218105], + [-0.064137791831655, -0.325377847425684, -0.158924039726607], + [ 0. , -0.343742581679188, -0.113925986025684], + [ 0.064137791831655, -0.325377847425684, -0.158924039726607], + [ 0.132325402795928, -0.290857984604968, -0.187067868218105], + [ 0.181481567104525, -0.243239316141725, -0.231284988892766], + [ 0.083999507750469, -0.239717753728704, -0.155256465640701], + [ 0. , -0.256058040176369, -0.0950619498899 ], + [-0.083999507750469, -0.239717753728704, -0.155256465640701], + [-0.181481567104525, -0.243239316141725, -0.231284988892766], + [-0.074036069749345, -0.250689938345682, -0.177346470406188], + [ 0. , -0.264945854681568, -0.112349967428413], + [ 0.074036069749345, -0.250689938345682, -0.177346470406188], + # Pupils and eyeball centers + [ 0.257990002632141, 0.276080012321472, -0.219998998939991], + [-0.257990002632141, 0.276080012321472, -0.219998998939991], + [ 0.257990002632141, 0.276080012321472, -0.324570998549461], + [-0.257990002632141, 0.276080012321472, -0.324570998549461] + ], np.float32) def __init__(self, featureType): self.featureType = featureType - self.face_3d = np.array([ - [ 0.4551769692672 , 0.300895790030204, -0.764429433974752], - [ 0.448998827123556, 0.166995837790733, -0.765143004071253], - [ 0.437431554952677, 0.022655479179981, -0.739267175112735], - [ 0.415033422928434, -0.088941454648772, -0.747947437846473], - [ 0.389123587370091, -0.232380029794684, -0.704788385327458], - [ 0.334630113904382, -0.361265387599081, -0.615587579236862], - [ 0.263725112132858, -0.460009725616771, -0.491479221041573], - [ 0.16241621322721 , -0.558037146073869, -0.339445180872282], - [ 0. , -0.621079019321682, -0.287294770748887], - [-0.16241621322721 , -0.558037146073869, -0.339445180872282], - [-0.263725112132858, -0.460009725616771, -0.491479221041573], - [-0.334630113904382, -0.361265387599081, -0.615587579236862], - [-0.389123587370091, -0.232380029794684, -0.704788385327458], - [-0.415033422928434, -0.088941454648772, -0.747947437846473], - [-0.437431554952677, 0.022655479179981, -0.739267175112735], - [-0.448998827123556, 0.166995837790733, -0.765143004071253], - [-0.4551769692672 , 0.300895790030204, -0.764429433974752], - [ 0.385529968662985, 0.402800553948697, -0.310031082540741], - [ 0.322196658344302, 0.464439136821772, -0.250558059367669], - [ 0.25409760441282 , 0.46420381416882 , -0.208177722146526], - [ 0.186875436782135, 0.44706071961879 , -0.145299823706503], - [ 0.120880983543622, 0.423566314072968, -0.110757158774771], - [-0.120880983543622, 0.423566314072968, -0.110757158774771], - [-0.186875436782135, 0.44706071961879 , -0.145299823706503], - [-0.25409760441282 , 0.46420381416882 , -0.208177722146526], - [-0.322196658344302, 0.464439136821772, -0.250558059367669], - [-0.385529968662985, 0.402800553948697, -0.310031082540741], - [ 0. , 0.293332603215811, -0.137582088779393], - [ 0. , 0.194828701837823, -0.069158109325951], - [ 0. , 0.103844017393155, -0.009151819844964], - [ 0. , 0. , 0. ], - [ 0.080626352317973, -0.041276068128093, -0.134161035564826], - [ 0.046439347377934, -0.057675223874769, -0.102990627164664], - [ 0. , -0.068753126205604, -0.090545348482397], - [-0.046439347377934, -0.057675223874769, -0.102990627164664], - [-0.080626352317973, -0.041276068128093, -0.134161035564826], - [ 0.315905195966084, 0.298337502555443, -0.285107407636464], - [ 0.275252345439353, 0.312721904921771, -0.244558251170671], - [ 0.176394511553111, 0.311907184376107, -0.219205360345231], - [ 0.131229723798772, 0.284447361805627, -0.234239149487417], - [ 0.184124948330084, 0.260179585304867, -0.226590776513707], - [ 0.279433549294448, 0.267363071770222, -0.248441437111633], - [-0.131229723798772, 0.284447361805627, -0.234239149487417], - [-0.176394511553111, 0.311907184376107, -0.219205360345231], - [-0.275252345439353, 0.312721904921771, -0.244558251170671], - [-0.315905195966084, 0.298337502555443, -0.285107407636464], - [-0.279433549294448, 0.267363071770222, -0.248441437111633], - [-0.184124948330084, 0.260179585304867, -0.226590776513707], - [ 0.121155252430729, -0.208988660580347, -0.160606287940521], - [ 0.041356305910044, -0.194484199722098, -0.096159882202821], - [ 0. , -0.205180167345702, -0.083299217789729], - [-0.041356305910044, -0.194484199722098, -0.096159882202821], - [-0.121155252430729, -0.208988660580347, -0.160606287940521], - [-0.132325402795928, -0.290857984604968, -0.187067868218105], - [-0.064137791831655, -0.325377847425684, -0.158924039726607], - [ 0. , -0.343742581679188, -0.113925986025684], - [ 0.064137791831655, -0.325377847425684, -0.158924039726607], - [ 0.132325402795928, -0.290857984604968, -0.187067868218105], - [ 0.181481567104525, -0.243239316141725, -0.231284988892766], - [ 0.083999507750469, -0.239717753728704, -0.155256465640701], - [ 0. , -0.256058040176369, -0.0950619498899 ], - [-0.083999507750469, -0.239717753728704, -0.155256465640701], - [-0.181481567104525, -0.243239316141725, -0.231284988892766], - [-0.074036069749345, -0.250689938345682, -0.177346470406188], - [ 0. , -0.264945854681568, -0.112349967428413], - [ 0.074036069749345, -0.250689938345682, -0.177346470406188], - # Pupils and eyeball centers - [ 0.257990002632141, 0.276080012321472, -0.219998998939991], - [-0.257990002632141, 0.276080012321472, -0.219998998939991], - [ 0.257990002632141, 0.276080012321472, -0.324570998549461], - [-0.257990002632141, 0.276080012321472, -0.324570998549461] - ], np.float32) + self.reset() diff --git a/facedetection.py b/facedetection.py index f3349a1..c3a3845 100644 --- a/facedetection.py +++ b/facedetection.py @@ -1,34 +1,35 @@ import numpy as np import cv2 +cv2.setNumThreads(6) def detect_faces(frame, model, detection_threshold ): - image = resizeImage(frame) + image = resizeImage(frame) - outputs, maxpool = model.faceDetection.run([], {'input': image}) - outputs = outputs[0] - maxpool = maxpool[0] - mask = outputs[0] == maxpool[0] - mask2 = outputs[1] > 0.15 - outputs[0] = outputs[0] * mask.astype(int) * mask2.astype(int) - faceLocation = np.argmax(outputs[0].flatten()) - x = faceLocation % 56 - y = faceLocation // 56 - if outputs[0, y, x] < detection_threshold: - return None - r = outputs[1, y, x] * 112. - results= (((x * 4) - r, (y * 4) - r, r*2,r*2)) - results = np.array(results).astype(np.float32) - results[[0,2]] *= frame.shape[1] / 224. - results[[1,3]] *= frame.shape[0] / 224. + outputs, maxpool = model.faceDetection.run([], {'input': image}) + outputs = outputs[0] + maxpool = maxpool[0] + mask = outputs[0] == maxpool[0] + mask2 = outputs[1] > 0.15 + outputs[0] = outputs[0] * mask.astype(int) * mask2.astype(int) + faceLocation = np.argmax(outputs[0].flatten()) + x = faceLocation % 56 + y = faceLocation // 56 + if outputs[0, y, x] < detection_threshold: + return None + r = outputs[1, y, x] * 112. + results= (((x * 4) - r, (y * 4) - r, r*2,r*2)) + results = np.array(results).astype(np.float32) + results[[0,2]] *= frame.shape[1] / 224. + results[[1,3]] *= frame.shape[0] / 224. - return results + return results def resizeImage(frame): mean = np.float32(np.array([-2.1179, -2.0357, -1.8044])) std = np.float32(np.array([0.0171, 0.0175, 0.0174])) targetimageSize = [224, 224] - image = cv2.resize(frame, targetimageSize, interpolation=cv2.INTER_LINEAR) * std + mean + image = cv2.resize(frame, targetimageSize, interpolation=cv2.INTER_CUBIC) * std + mean image = np.expand_dims(image, 0) image = np.transpose(image, (0,3,1,2)) return image diff --git a/facetracker.py b/facetracker.py index e161e07..223569c 100644 --- a/facetracker.py +++ b/facetracker.py @@ -26,14 +26,13 @@ parser.add_argument("--model", type=int, help="This can be used to select the tr parser.add_argument("--preview", type=int, help="Preview the frames sent to the tracker", default=0) parser.add_argument("--feature-type", type=int, help="Sets which version of feature extraction is used. 0 is my new version that works well for me and allows for some customization, 1 is EmilianaVT's version aka, normal OpenSeeFace operation", default=0, choices=[0, 1]) parser.add_argument("--numpy-threads", type=int, help="Numer of threads Numpy can use, doesn't seem to effect much", default=1) -parser.add_argument("-T","--threads", type=int, help="Numer of threads used for landmark detection. Default is 1 (~15ms per frame on my computer), 2 gets slightly faster frames (~10ms on my computer), more than 2 doesn't seem to help much", default=1) +parser.add_argument("-T","--threads", type=int, help="Numer of threads used for landmark detection. Default is 1 (~15ms per frame on my computer), 2 gets slightly faster frames (~10ms on my computer), more than 2 doesn't seem to help much", default=2) parser.add_argument("-v", "--visualize", type=int, help="Set this to 1 to visualize the tracking", default=0) args = parser.parse_args() #processing args -#For Vtube Studio: ip 127.0.0.1 port 11573 target_ip = args.ip target_port = args.port fps = args.fps @@ -53,7 +52,8 @@ silent = (args.silent == 1) if height > 480: frameQueueSize = 2 else: - frameQueueSize = 1 + frameQueueSize = 2 + frameQueue = multiprocessing.Queue(maxsize=frameQueueSize) faceQueue = multiprocessing.Queue(maxsize=1) packetQueue = queue.Queue() @@ -65,9 +65,7 @@ packetSenderThread.daemon = True packetSenderThread.start() #creating processes -#not a very active process, it's blocked by the main thread 90% of the time -#but putting it in it's own process means there's a little bit less occupying the same core as the main thread -#which helps when the system is under load + webcamProcess = multiprocessing.Process(target=webcam.startProcess,args = (frameQueue, faceQueue, fps, targetBrightness, width, height, mirror_input, )) webcamProcess.daemon = True webcamProcess.start() @@ -119,23 +117,23 @@ try: if previewFrameQueue.qsize() < 1: previewFrameQueue.put(frame) - face, face_center, face_radius = tracker.predict(frame) + faceInfo, face = tracker.predict(frame) - if face is not None: - packet = VTS.preparePacket(face) + if faceInfo is not None: + packet = VTS.preparePacket(faceInfo) if faceQueue.qsize() < 1: - faceQueue.put([face_center, face_radius]) + faceQueue.put(face) if visualizeFlag: - preview.visualize(frame, face, previewFrameQueue, face_center, face_radius) + preview.visualize(frame, faceInfo, previewFrameQueue, face_center, face_radius) duration = time.perf_counter() - frame_get total_active_time += duration peak_frame_time = max(peak_frame_time, duration) - - - sleepTimer = max(time.perf_counter() - frame_get, sleepTimer) - time.sleep(max(peak_frame_time - duration, 0)) - sleepTimer -= 0.0005 + if duration < target_duration: + time.sleep(target_duration - duration) + else: + #all caps because if this is consistently happening it's an issue + messageQueue.put("CANNOT MAINTAIN FRAMERATE!") peak_time_between = max(peak_time_between, time.perf_counter() -frame_start) total_run_time += time.perf_counter() -frame_start diff --git a/image.py b/image.py new file mode 100644 index 0000000..e69de29 diff --git a/landmarks.py b/landmarks.py index 882fe20..0ce3c72 100644 --- a/landmarks.py +++ b/landmarks.py @@ -1,6 +1,7 @@ import numpy as np import math import cv2 +cv2.setNumThreads(6) def matrix_to_quaternion(m): t = 0.0 diff --git a/preview.py b/preview.py index eb9e401..f55b866 100644 --- a/preview.py +++ b/preview.py @@ -1,5 +1,6 @@ import cv2 import multiprocessing +cv2.setNumThreads(6) #this is it's own file just for ceanlieness diff --git a/tracker.py b/tracker.py index b4de59d..9dbb8f3 100644 --- a/tracker.py +++ b/tracker.py @@ -2,6 +2,7 @@ import os import numpy as np import onnxruntime import cv2 +cv2.setNumThreads(6) import time import facedetection import eyes @@ -9,27 +10,34 @@ import landmarks import face + #this file is so much smaller than it used to be, lol #I moved most of the functionality into separate files so they're easier to work with -def clamp_to_im(pt, w, h): #8 times per frame, but that only accounts for 0.005ms - x=max(pt[0],0) - y=max(pt[1],0) +def clamp_to_im(pt, w, h): + x = pt[0] + y = pt[1] + if x < 0: + x = 0 + if y < 0: + y = 0 if x >= w: x = w-1 if y >= h: y = h-1 - return (int(x),int(y+1)) + return (int(x), int(y+1)) class Models(): + + models = [ + "lm_model0_opt.onnx", + "lm_model1_opt.onnx", + "lm_model2_opt.onnx", + "lm_model3_opt.onnx", + "lm_model4_opt.onnx"] + def __init__(self, threads, model_type=3): self.model_type = model_type - self.models = [ - "lm_model0_opt.onnx", - "lm_model1_opt.onnx", - "lm_model2_opt.onnx", - "lm_model3_opt.onnx", - "lm_model4_opt.onnx"] model = self.models[self.model_type] model_base_path = os.path.join(os.path.dirname(__file__), os.path.join("models")) @@ -48,6 +56,9 @@ class Models(): self.gazeTracker = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options, providers=providersList) class Tracker(): + targetimageSize = [224, 224] + std = np.float32(np.array([0.0171, 0.0175, 0.0174])) + mean = np.float32(np.array([-2.1179, -2.0357, -1.8044])) def __init__(self, width, height, featureType, threads, model_type=3, detection_threshold=0.6, threshold=0.6, silent=False): self.detection_threshold = detection_threshold @@ -55,9 +66,6 @@ class Tracker(): self.model = Models(threads, model_type = model_type) # Image normalization constants - self.mean = np.float32(np.array([-2.1179, -2.0357, -1.8044])) - self.std = np.float32(np.array([0.0171, 0.0175, 0.0174])) - self.targetimageSize = [224, 224] self.width = width self.height = height @@ -75,12 +83,10 @@ class Tracker(): duration_pp = 0.0 x,y,w,h = self.face - wint = int(w * 0.1) - hint = int(h * 0.125) - crop_x1 = x - wint - crop_y1 = y - hint - crop_x2 = x + w + wint - crop_y2 = y + h + hint + crop_x1 = x - (w * 0.1) + crop_y1 = y - (h * 0.125) + crop_x2 = x + w + (w * 0.1) + crop_y2 = y + h + (h * 0.125) crop_x1, crop_y1 = clamp_to_im((crop_x1, crop_y1), self.width, self.height) crop_x2, crop_y2 = clamp_to_im((crop_x2, crop_y2), self.width, self.height) @@ -104,7 +110,7 @@ class Tracker(): self.face = None duration = (time.perf_counter() - start) * 1000 print(f"Took {duration:.2f}ms") - return None, None, None + return None, None def predict(self, frame): start = time.perf_counter() @@ -146,13 +152,13 @@ class Tracker(): if face_info.success: face_info.adjust_3d() lms = face_info.lms[:, 0:2] - x1, y1 = lms[0:66].min(0) - x2, y2 = lms[0:66].max(0) - self.face = [y1, x1, y2 - y1, x2 - x1] + y1, x1 = lms[0:66].min(0) + y2, x2 = lms[0:66].max(0) + self.face = [x1, y1, x2 - x1, y2 - y1] duration_pnp += 1000 * (time.perf_counter() - start_pnp) duration = (time.perf_counter() - start) * 1000 print(f"Took {duration:.2f}ms (detect: {duration_fd:.2f}ms, crop: {duration_pp:.2f}ms, track: {duration_model:.2f}ms, 3D points: {duration_pnp:.2f}ms)") - return face_info, self.EyeTracker.faceCenter, self.EyeTracker.faceRadius + return face_info, self.face #Combined multiple failures into one catch all exit return self.early_exit("Face info not valid", start) diff --git a/vts.py b/vts.py index a1944ff..8dfed2c 100644 --- a/vts.py +++ b/vts.py @@ -3,13 +3,14 @@ import socket import struct class VTS(): + + features = ["eye_l", "eye_r", "eyebrow_steepness_l", "eyebrow_updown_l", "eyebrow_quirk_l", "eyebrow_steepness_r", "eyebrow_updown_r", "eyebrow_quirk_r", "mouth_corner_updown_l", "mouth_corner_inout_l", "mouth_corner_updown_r", "mouth_corner_inout_r", "mouth_open", "mouth_wide"] def __init__(self, target_ip, target_port, silent, height, width, packetQueue): self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.target_ip = target_ip self.target_port = target_port self.silent = silent - self.features = ["eye_l", "eye_r", "eyebrow_steepness_l", "eyebrow_updown_l", "eyebrow_quirk_l", "eyebrow_steepness_r", "eyebrow_updown_r", "eyebrow_quirk_r", "mouth_corner_updown_l", "mouth_corner_inout_l", "mouth_corner_updown_r", "mouth_corner_inout_r", "mouth_open", "mouth_wide"] self.width = width self.height = height self.packetQueue = packetQueue diff --git a/webcam.py b/webcam.py index 1b66e11..538284b 100644 --- a/webcam.py +++ b/webcam.py @@ -3,11 +3,24 @@ import os os.environ["OMP_NUM_THREADS"] = str(1) import dshowcapture import cv2 +cv2.setNumThreads(6) import multiprocessing import time import numpy as np import math +def clamp_to_im(pt, w, h): + x = pt[0] + y = pt[1] + if x < 0: + x = 0 + if y < 0: + y = 0 + if x >= w: + x = w-1 + if y >= h: + y = h-1 + return (int(x), int(y+1)) #this is just here for conveience def startProcess(frameQueue, faceQueue, fps, targetBrightness, width, height, mirrorInput): @@ -43,6 +56,7 @@ class Webcam(): self.frame = None def start(self): + while self.cap.isOpened(): frameStart= time.perf_counter() totalFrameLatency = time.perf_counter() @@ -91,17 +105,20 @@ class Webcam(): #calculate the ideal gamma based on the brightness of the user's face #kind of winging it with the math, but it's a lot better than calculating based on the whole frame def updateGamma(self): - face_center, face_radius = self.faceQueue.get() + face = self.faceQueue.get() - y1 = int(face_center[1] - face_radius[1]) - y2 = int(face_center[1] + face_radius[1]) - x1 = int(face_center[0] - face_radius[0]) - x2 = int(face_center[0] + face_radius[0]) + x,y,w,h = face + crop_x1 = x + crop_y1 = y + crop_x2 = x + w + crop_y2 = y + h - #I was doing random sampling of the image brightness - #and that's probably brighter for the whole frame - #but for just the face this is pretty fast - averageBrightness = np.mean(self.brightnessFrame[y1:y2,x1:x2])/256 + crop_x1, crop_y1 = clamp_to_im((crop_x1, crop_y1), self.width, self.height) + crop_x2, crop_y2 = clamp_to_im((crop_x2, crop_y2), self.width, self.height) + + croppedFace = self.brightnessFrame[crop_y1:crop_y2, crop_x1:crop_x2] + + averageBrightness = np.mean(croppedFace)/256 self.gamma = math.log(self.targetBrightness, averageBrightness) return