moved visualization to the preview thread, reduced latency, added --threads functionality
This commit is contained in:
Cowwan
parent
8236a8acd0
commit
60886f9f95
5 changed files with 59 additions and 72 deletions
|
|
@ -1,8 +1,4 @@
|
|||
#import copy
|
||||
import os
|
||||
#limits the threads used by numpy to 1
|
||||
#setting this and the landmarks model to use 1 means the cpu usage is pretty low
|
||||
|
||||
os.environ["OMP_NUM_THREADS"] = str(1)
|
||||
import argparse
|
||||
import traceback
|
||||
|
|
@ -14,7 +10,6 @@ import multiprocessing
|
|||
import webcam
|
||||
import vts
|
||||
import preview
|
||||
import cv2
|
||||
|
||||
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument("-i", "--ip", help="Set IP address for sending tracking data", default="127.0.0.1")
|
||||
|
|
@ -31,33 +26,12 @@ parser.add_argument("--model", type=int, help="This can be used to select the tr
|
|||
parser.add_argument("--preview", type=int, help="Preview the frames sent to the tracker", default=0)
|
||||
parser.add_argument("--feature-type", type=int, help="Sets which version of feature extraction is used. 0 is my new version that works well for me and allows for some customization, 1 is EmilianaVT's version aka, normal OpenSeeFace operation", default=0, choices=[0, 1])
|
||||
parser.add_argument("--numpy-threads", type=int, help="Numer of threads Numpy can use, doesn't seem to effect much", default=1)
|
||||
parser.add_argument("--landmark-detection-threads", type=int, help="Numer of threads used for landmark detection. Default is 1 (~15ms per frame on my computer), 2 gets slightly faster frames (~10ms on my computer), more than 2 doesn't seem to help much", default=1)
|
||||
parser.add_argument("-T","--threads", type=int, help="Numer of threads used for landmark detection. Default is 1 (~15ms per frame on my computer), 2 gets slightly faster frames (~10ms on my computer), more than 2 doesn't seem to help much", default=1)
|
||||
parser.add_argument("-v", "--visualize", type=int, help="Set this to 1 to visualize the tracking", default=0)
|
||||
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
def visualize(frame, face, previewFrameQueue, face_center, face_radius):
|
||||
|
||||
y1 = int(face_center[1] - face_radius[1])
|
||||
y2 = int(face_center[1] + face_radius[1])
|
||||
x1 = int(face_center[0] - face_radius[0])
|
||||
x2 = int(face_center[0] + face_radius[0])
|
||||
bbox = (y1, x1, y2 - y1, x2 - x1)
|
||||
frame = cv2.putText(frame, str(face.id), (int(bbox[1]), int(bbox[0])), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (255,0,255))
|
||||
frame = cv2.putText(frame, f"{face.conf:.4f}", (int(bbox[1] + 18), int(bbox[0] - 6)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,0,255))
|
||||
for pt_num, (x,y,c) in enumerate(face.lms):
|
||||
x = int(x + 0.5)
|
||||
y = int(y + 0.5)
|
||||
frame = cv2.putText(frame, str(pt_num), (int(y), int(x)), cv2.FONT_HERSHEY_SIMPLEX, 0.25, (255,255,0))
|
||||
color = (0, 255, 0)
|
||||
if pt_num >= 66:
|
||||
color = (255, 255, 0)
|
||||
if not (x < 0 or y < 0 or x >= height or y >= width):
|
||||
cv2.circle(frame, (y, x), 1, color, -1)
|
||||
previewFrameQueue.put(frame)
|
||||
|
||||
|
||||
#processing args
|
||||
#For Vtube Studio: ip 127.0.0.1 port 11573
|
||||
target_ip = args.ip
|
||||
|
|
@ -69,6 +43,7 @@ previewFlag = (args.preview == 1)
|
|||
mirror_input = args.mirror_input
|
||||
featureType = args.feature_type
|
||||
visualizeFlag = (args.visualize == 1)
|
||||
threads = args.threads
|
||||
|
||||
target_duration = 1. / fps
|
||||
targetBrightness = 0.55
|
||||
|
|
@ -115,8 +90,10 @@ total_camera_latency = 0.0
|
|||
sleepTimer = 0.0
|
||||
|
||||
failures = 0
|
||||
peakTotalLatency = 0.0
|
||||
totalTotalLatency = 0.0
|
||||
|
||||
tracker = Tracker(width, height, featureType, threshold=args.threshold, silent=silent, model_type=args.model, detection_threshold=args.detection_threshold)
|
||||
tracker = Tracker(width, height, featureType, threads, threshold=args.threshold, silent=silent, model_type=args.model, detection_threshold=args.detection_threshold)
|
||||
|
||||
#don't start until the webcam is ready, then give it a little more time to fill it's buffer
|
||||
while frameQueue.qsize() < 1:
|
||||
|
|
@ -130,49 +107,35 @@ try:
|
|||
packet = None
|
||||
frame_start = time.perf_counter()
|
||||
frame_count += 1
|
||||
if True:
|
||||
frame, camera_latency, totalFrameLatency = frameQueue.get()
|
||||
frame_get = time.perf_counter()
|
||||
#If I don't wait a few frames to start tracking I get wild peak frame times, like 500ms
|
||||
if frame_count > 5:
|
||||
peak_camera_latency = max(camera_latency, peak_camera_latency)
|
||||
total_camera_latency+= camera_latency
|
||||
frame, camera_latency, totalFrameLatency = frameQueue.get()
|
||||
frame_get = time.perf_counter()
|
||||
#If I don't wait a few frames to start tracking I get wild peak frame times, like 500ms
|
||||
if frame_count > 5:
|
||||
peak_camera_latency = max(camera_latency, peak_camera_latency)
|
||||
total_camera_latency+= camera_latency
|
||||
|
||||
if previewFlag:
|
||||
#there's no reason to ever wait on this process or give it much of a queue
|
||||
if previewFrameQueue.qsize() < 1:
|
||||
previewFrameQueue.put(frame)
|
||||
|
||||
try:
|
||||
face, face_center, face_radius = tracker.predict(frame)
|
||||
failures = 0
|
||||
except Exception as e:
|
||||
if e.__class__ == KeyboardInterrupt:
|
||||
if not silent:
|
||||
print("Quitting")
|
||||
break
|
||||
traceback.print_exc()
|
||||
failures += 1
|
||||
if failures > 30:
|
||||
break
|
||||
if previewFlag:
|
||||
#there's no reason to ever wait on this process or give it much of a queue
|
||||
if previewFrameQueue.qsize() < 1:
|
||||
previewFrameQueue.put(frame)
|
||||
|
||||
face, face_center, face_radius = tracker.predict(frame)
|
||||
|
||||
if face is not None:
|
||||
packet = VTS.preparePacket(face)
|
||||
if faceQueue.qsize() < 1:
|
||||
faceQueue.put([face_center, face_radius])
|
||||
if visualizeFlag:
|
||||
visualize(frame, face, previewFrameQueue, face_center, face_radius)
|
||||
preview.visualize(frame, face, previewFrameQueue, face_center, face_radius)
|
||||
|
||||
duration = time.perf_counter() - frame_get
|
||||
|
||||
total_active_time += duration
|
||||
peak_frame_time = max(peak_frame_time, duration)
|
||||
|
||||
|
||||
sleepTimer = max(time.perf_counter() - frame_get, sleepTimer)
|
||||
time.sleep(max(peak_frame_time - duration, 0))
|
||||
sleepTimer -= 0.0001
|
||||
sleepTimer -= 0.0005
|
||||
|
||||
peak_time_between = max(peak_time_between, time.perf_counter() -frame_start)
|
||||
total_run_time += time.perf_counter() -frame_start
|
||||
|
|
@ -182,7 +145,9 @@ try:
|
|||
packetQueue.put(packet)
|
||||
else:
|
||||
print("No data sent to VTS")
|
||||
print(time.perf_counter() - totalFrameLatency)
|
||||
if frame_count > 5:
|
||||
peakTotalLatency = max(peakTotalLatency, (time.perf_counter() - totalFrameLatency))
|
||||
totalTotalLatency += time.perf_counter() - totalFrameLatency
|
||||
|
||||
except KeyboardInterrupt:
|
||||
if not silent:
|
||||
|
|
@ -190,6 +155,9 @@ except KeyboardInterrupt:
|
|||
|
||||
#printing statistics on close
|
||||
#it makes identifying problems easier
|
||||
|
||||
print(f"Peak latency: {(peakTotalLatency*1000):.3f}ms")
|
||||
print(f"Average latency: {(totalTotalLatency*1000/(frame_count-5)):.3f}ms")
|
||||
#480p generally results in camera latencies between 0.5ms and 2.0ms
|
||||
print(f"Peak camera latency: {(peak_camera_latency*1000):.3f}ms")
|
||||
print(f"Average camera latency: {(total_camera_latency*1000/(frame_count-5)):.3f}ms")
|
||||
|
|
|
|||
|
|
@ -16,5 +16,5 @@ python3 -m venv .venv
|
|||
source .venv/bin/activate
|
||||
|
||||
#the openseeface command
|
||||
python3 facetracker.py --fps 30 --ip 127.0.0.1 --port 11573 --model 3 -W 640 -H 480 #-W 1920 -H 1080 #-W 640 -H 480 #--preview 1
|
||||
python3 facetracker.py --fps 30 --ip 127.0.0.1 --port 11573 --model 3 -W 640 -H 480 #-W 1920 -H 1080 #-W 640 -H 480 #--preview 1
|
||||
|
||||
|
|
|
|||
19
preview.py
19
preview.py
|
|
@ -19,3 +19,22 @@ class Preview():
|
|||
cv2.imshow("test",cv2.cvtColor(frame, cv2.COLOR_RGB2BGR))
|
||||
cv2.waitKey(1)
|
||||
|
||||
def visualize(frame, face, previewFrameQueue, face_center, face_radius):
|
||||
|
||||
y1 = int(face_center[1] - face_radius[1])
|
||||
y2 = int(face_center[1] + face_radius[1])
|
||||
x1 = int(face_center[0] - face_radius[0])
|
||||
x2 = int(face_center[0] + face_radius[0])
|
||||
bbox = (y1, x1, y2 - y1, x2 - x1)
|
||||
frame = cv2.putText(frame, str(face.id), (int(bbox[1]), int(bbox[0])), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (255,0,255))
|
||||
frame = cv2.putText(frame, f"{face.conf:.4f}", (int(bbox[1] + 18), int(bbox[0] - 6)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,0,255))
|
||||
for pt_num, (x,y,c) in enumerate(face.lms):
|
||||
x = int(x + 0.5)
|
||||
y = int(y + 0.5)
|
||||
frame = cv2.putText(frame, str(pt_num), (int(y), int(x)), cv2.FONT_HERSHEY_SIMPLEX, 0.25, (255,255,0))
|
||||
color = (0, 255, 0)
|
||||
if pt_num >= 66:
|
||||
color = (255, 255, 0)
|
||||
if not (x < 0 or y < 0 or x >= height or y >= width):
|
||||
cv2.circle(frame, (y, x), 1, color, -1)
|
||||
previewFrameQueue.put(frame)
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ def clamp_to_im(pt, w, h): #8 times per frame, but that only accounts for 0.005m
|
|||
return (int(x),int(y+1))
|
||||
|
||||
class Models():
|
||||
def __init__(self, model_type=3):
|
||||
def __init__(self, threads, model_type=3):
|
||||
self.model_type = model_type
|
||||
self.models = [
|
||||
"lm_model0_opt.onnx",
|
||||
|
|
@ -36,7 +36,7 @@ class Models():
|
|||
providersList = onnxruntime.capi._pybind_state.get_available_providers()
|
||||
options = onnxruntime.SessionOptions()
|
||||
options.inter_op_num_threads = 1
|
||||
options.intra_op_num_threads = 1
|
||||
options.intra_op_num_threads = threads
|
||||
options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
|
||||
options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
options.log_severity_level = 3
|
||||
|
|
@ -48,11 +48,11 @@ class Models():
|
|||
self.gazeTracker = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options, providers=providersList)
|
||||
|
||||
class Tracker():
|
||||
def __init__(self, width, height, featureType, model_type=3, detection_threshold=0.6, threshold=0.6, silent=False):
|
||||
def __init__(self, width, height, featureType, threads, model_type=3, detection_threshold=0.6, threshold=0.6, silent=False):
|
||||
|
||||
self.detection_threshold = detection_threshold
|
||||
self.EyeTracker = eyes.EyeTracker()
|
||||
self.model = Models(model_type = model_type)
|
||||
self.model = Models(threads, model_type = model_type)
|
||||
|
||||
# Image normalization constants
|
||||
self.mean = np.float32(np.array([-2.1179, -2.0357, -1.8044]))
|
||||
|
|
|
|||
22
webcam.py
22
webcam.py
|
|
@ -40,19 +40,19 @@ class Webcam():
|
|||
self.width = width #unused, but it seemed useful to have around
|
||||
self.height = height #unused, but it seemed useful to have around
|
||||
self.ret = 0
|
||||
self.frame = None
|
||||
|
||||
def start(self):
|
||||
while self.cap.isOpened():
|
||||
frameStart= time.perf_counter()
|
||||
totalFrameLatency = time.perf_counter()
|
||||
frame = self.getFrame()
|
||||
self.getFrame()
|
||||
self.cameraLatency = time.perf_counter() - frameStart
|
||||
if self.ret:
|
||||
frame = self.applyGamma(frame)
|
||||
self.applyGamma()
|
||||
if self.mirror:
|
||||
frame = cv2.flip(frame, 1)
|
||||
#this line is where this process sits 90% of the time
|
||||
self.frameQueue.put([frame, self.cameraLatency, totalFrameLatency])
|
||||
self.frame = cv2.flip(self.frame, 1)
|
||||
self.frameQueue.put([self.frame, self.cameraLatency, totalFrameLatency])
|
||||
if self.faceQueue.qsize() > 0:
|
||||
self.updateGamma()
|
||||
sleepTime = self.targetFrameTime - (time.perf_counter() - frameStart)
|
||||
|
|
@ -67,15 +67,15 @@ class Webcam():
|
|||
#keeping track of how long it takes to get frames from the webcam
|
||||
#because that one line is the cause of 90% of late frames
|
||||
cameraStart = time.perf_counter()
|
||||
self.ret, frame = self.cap.read()
|
||||
self.ret, self.frame = self.cap.read()
|
||||
self.cameraLatency = time.perf_counter() - cameraStart
|
||||
return(frame)
|
||||
return
|
||||
|
||||
#Applies a gamma curve to the frame
|
||||
#Uses a lookup table because that's way faster than actually calculating every pixel
|
||||
#Immensely improves tracking in low light situations
|
||||
def applyGamma(self, frame):
|
||||
img_yuv = cv2.cvtColor(frame, cv2.COLOR_BGR2YUV)
|
||||
def applyGamma(self):
|
||||
img_yuv = cv2.cvtColor(self.frame, cv2.COLOR_BGR2YUV)
|
||||
#saving a copy of the brightness channel so I can use it to adjust the gamma based on where the face is in that frame
|
||||
self.brightnessFrame = img_yuv[:,:,0].copy()
|
||||
#building a lookup table on the fly
|
||||
|
|
@ -84,8 +84,8 @@ class Webcam():
|
|||
lookupTable = np.power(loopupTable, self.gamma)*255
|
||||
img_yuv[:,:,0] = lookupTable[img_yuv[:,:,0]]
|
||||
#I convert the image to RBG here because it was getting repeatedly converted in the face tracking
|
||||
frame = cv2.cvtColor(img_yuv, cv2.COLOR_YUV2RGB)
|
||||
return frame
|
||||
self.frame = cv2.cvtColor(img_yuv, cv2.COLOR_YUV2RGB)
|
||||
|
||||
|
||||
|
||||
#calculate the ideal gamma based on the brightness of the user's face
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue