Did a bunch to make webcams work at higher resolutions without issues

now I'm sending stale frame data instead of none if the camera falls behind
and I have automatic switches to help with video higher than 480p on linux
and I adjusted the timings to keep the camera from falling behind

facetracker.py

@@ -20,7 +20,7 @@ parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFo
 parser.add_argument("-i", "--ip", help="Set IP address for sending tracking data", default="127.0.0.1")
 parser.add_argument("-p", "--port", type=int, help="Set port for sending tracking data", default=11573)
 parser.add_argument("-W", "--width", type=int, help="Set raw RGB width", default=640)
-parser.add_argument("-H", "--height", type=int, help="Set raw RGB height", default=360)
+parser.add_argument("-H", "--height", type=int, help="Set raw RGB height", default=480)
 parser.add_argument("-F", "--fps", type=int, help="Set camera frames per second", default=24)
 parser.add_argument("-c", "--capture", help="Set camera ID (0, 1...) or video file", default=0)
 parser.add_argument("-M", "--mirror-input", action="store_true", help="Process a mirror image of the input video")
@@ -70,25 +70,15 @@ mirror_input = args.mirror_input
 featureType = args.feature_type
 visualizeFlag = (args.visualize == 1)
 
-#I make a minor tweak to the framerate to help the camera out
-#it's an inpercetible amount of time per frame
-#but my hope is it keeps things from riding up against what the webcam can put out
-target_duration = 1. / (fps-0.01)
+target_duration = 1. / fps
 targetBrightness = 0.55
 silent = (args.silent == 1)
 
-
-#increating the webcam queue to compensate for longer frame times at different settings
-frameQueueSize = 1
-if height > 480:
-    frameQueueSize = 3
-    #My testing has shown that it can take upwards of 80ms when capturing images above 480p
-    #each frame is 33ms, so I'd need 3 frames, or 99ms to cover that without skipping a frame
-if fps > 30:
-    frameQueueSize = frameQueueSize * 2
-    #I'm just going to assume the webcam does not work well above 30fps
-
 #creating queues
+if height > 480:
+    frameQueueSize = 2
+else:
+    frameQueueSize = 1
 frameQueue = multiprocessing.Queue(maxsize=frameQueueSize)
 faceQueue = multiprocessing.Queue(maxsize=1)
 packetQueue = queue.Queue()
@@ -128,12 +118,13 @@ tracker = Tracker(width, height, featureType,  threshold=args.threshold, silent=
 
 #don't start until the webcam is ready
 while frameQueue.qsize() < 1:
-    time.sleep(0.33)
+    time.sleep(0.2)
+face = None
+
 try:
     while True:
         #clearing these so there's no stale data to confuse the checks
         packet = None
-        face = None
         frame_start = time.perf_counter()
         frame_count += 1
         #if the camera process doesn't have a frame for us we just skip everything
@@ -151,6 +142,7 @@ try:
                 #there's no reason to ever wait on this process or give it much of a queue
                 if previewFrameQueue.qsize() < 1:
                     previewFrameQueue.put(frame)
+
             try:
                 face, face_center, face_radius = tracker.predict(frame)
                 failures = 0
@@ -164,14 +156,13 @@ try:
                 if failures > 30:
                     break
 
-            if face is not None:
-                packet = VTS.preparePacket(face)
-                if faceQueue.qsize() < 1:
-                    faceQueue.put([face_center, face_radius])
-                if visualizeFlag:
-                    visualize(frame, face, previewFrameQueue, face_center, face_radius)
-        else:
-            print("Camera behind, skipping frame")
+
+        if face is not None:
+            packet = VTS.preparePacket(face)
+            if faceQueue.qsize() < 1:
+                faceQueue.put([face_center, face_radius])
+            if visualizeFlag:
+                visualize(frame, face, previewFrameQueue, face_center, face_radius)
 
         duration = time.perf_counter() - frame_start
 

facetracking.sh

@@ -16,5 +16,5 @@ python3 -m venv .venv
 source .venv/bin/activate
 
 #the openseeface command
-python3 facetracker.py  --fps 30 --ip 127.0.0.1 --port 11573 --model 3  -W 640 -H 480 #--preview 1
+python3 facetracker.py  --fps 30 --ip 127.0.0.1 --port 11573 --model 3 -W 1920 -H 1080 #-W 1920 -H 1080  #-W 640 -H 480 #--preview 1
 

webcam.py

@@ -1,4 +1,6 @@
 import os
+
+os.environ["OMP_NUM_THREADS"] = str(1)
 import dshowcapture
 import cv2
 import multiprocessing
@@ -20,15 +22,16 @@ class Webcam():
             self.cap = cv2.VideoCapture(0)
         #tbh, these settings will make things way slower and frametimes will suffer
         #but they're the only way frames bigger than 480p will work
-        if height > 480 and os.name != 'nt':
+        if height > 480:
             self.cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*"MJPG"))
-            self.cap.set(38, 4)
         else:
-            self.cap.set(38, 2)
+            self.cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*"YUYV"))
+
+        self.cap.set(38, 2)
         self.cap.set(3, width)
         self.cap.set(4, height)
         self.cap.set(cv2.CAP_PROP_FPS, fps)
-        self.targetFrameTime = 1. / (fps-0.01)
+        self.targetFrameTime = 1. / fps
         self.gamma = 0.7
         self.mirror = mirrorInput
         self.frameQueue = frameQueue #outgoing frames from the webcam
@@ -38,20 +41,19 @@ class Webcam():
         self.targetBrightness = targetBrightness    #the target average brightness of the face, used in gamma calculations
         self.width = width  #unused, but it seemed useful to have around
         self.height = height    #unused, but it seemed useful to have around
-        self.frame = None
         self.ret = 0
 
     def start(self):
         while self.cap.isOpened():
             frameStart= time.perf_counter()
-            self.getFrame()
+            frame = self.getFrame()
             self.cameraLatency = time.perf_counter() - frameStart
             if self.ret:
-                self.applyGamma()
+                frame = self.applyGamma(frame)
                 if self.mirror:
-                    self.frame = cv2.flip(self.frame, 1)
+                    frame = cv2.flip(frame, 1)
                 #this line is where this process sits 90% of the time
-                self.frameQueue.put([self.frame, self.cameraLatency])
+                self.frameQueue.put([frame, self.cameraLatency])
                 if self.faceQueue.qsize() > 0:
                     self.updateGamma()
             sleepTime = self.targetFrameTime - (time.perf_counter() - frameStart)
@@ -66,15 +68,15 @@ class Webcam():
         #keeping track of how long it takes to get frames from the webcam
         #because that one line is the cause of 90% of late frames
         cameraStart = time.perf_counter()
-        self.ret, self.frame = self.cap.read()
+        self.ret, frame = self.cap.read()
         self.cameraLatency = time.perf_counter() - cameraStart
-        return
+        return(frame)
 
     #Applies a gamma curve to the frame
     #Uses a lookup table because that's way faster than actually calculating every pixel
     #Immensely improves tracking in low light situations
-    def applyGamma(self):
-        img_yuv = cv2.cvtColor(self.frame, cv2.COLOR_BGR2YUV)
+    def applyGamma(self, frame):
+        img_yuv = cv2.cvtColor(frame, cv2.COLOR_BGR2YUV)
         #saving a copy of the brightness channel so I can use it to adjust the gamma based on where the face is in that frame
         self.brightnessFrame = img_yuv[:,:,0].copy()
         #building a lookup table on the fly
@@ -83,8 +85,8 @@ class Webcam():
         lookupTable = np.power(loopupTable, self.gamma)*255
         img_yuv[:,:,0] = lookupTable[img_yuv[:,:,0]]
         #I convert the image to RBG here because it was getting repeatedly converted in the face tracking
-        self.frame = cv2.cvtColor(img_yuv, cv2.COLOR_YUV2RGB)
-        return
+        frame = cv2.cvtColor(img_yuv, cv2.COLOR_YUV2RGB)
+        return frame
 
 
     #calculate the ideal gamma based on the brightness of the user's face