removed the message queue and thread, it was a silly idea
This commit is contained in:
Cowwan
parent
31d16b20c0
commit
a6a9ae0a88
6 changed files with 21 additions and 53 deletions
10
eyes.py
10
eyes.py
|
|
@ -142,14 +142,13 @@ class EyeTracker():
|
|||
self.rightEye = Eye(0)
|
||||
self.mean = np.float32(np.array([-2.1179, -2.0357, -1.8044]))
|
||||
self.std = np.float32(np.array([0.0171, 0.0175, 0.0174]))
|
||||
self.faceFrame = None
|
||||
self.offset = None
|
||||
self.faceCenter = None
|
||||
self.faceRadius = None
|
||||
|
||||
def get_eye_state(self, models, frame, lms):
|
||||
|
||||
lms = self.extract_face(frame, np.array(lms)[:,0:2][:,::-1])
|
||||
lms, faceFrame = self.extract_face(frame, np.array(lms)[:,0:2][:,::-1])
|
||||
|
||||
self.rightEye.offset = self.offset
|
||||
self.leftEye.offset = self.offset
|
||||
|
|
@ -158,8 +157,8 @@ class EyeTracker():
|
|||
self.leftEye.innerPoint = lms[45,0:2]
|
||||
self.leftEye.outerPoint = lms[42,0:2]
|
||||
|
||||
self.rightEye.prepare_eye(self.faceFrame)
|
||||
self.leftEye.prepare_eye(self.faceFrame)
|
||||
self.rightEye.prepare_eye(faceFrame)
|
||||
self.leftEye.prepare_eye(faceFrame)
|
||||
|
||||
if self.rightEye.image is None or self.leftEye.image is None:
|
||||
return [[1.,0.,0.,0.],[1.,0.,0.,0.]] #Early exit if one of the eyes doesn't have data
|
||||
|
|
@ -182,5 +181,4 @@ class EyeTracker():
|
|||
x2, y2 = clamp_to_im(self.faceCenter + self.faceRadius + 1, h, w)
|
||||
self.offset = np.array((x1, y1))
|
||||
lms = (lms[:, 0:2] - self.offset).astype(int)
|
||||
self.faceFrame = frame[y1:y2, x1:x2]
|
||||
return lms
|
||||
return lms, frame[y1:y2, x1:x2]
|
||||
|
|
|
|||
|
|
@ -1,10 +1,6 @@
|
|||
import numpy as np
|
||||
import cv2
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def detect_faces(frame, model, detection_threshold ):
|
||||
|
||||
image = resizeImage(frame)
|
||||
|
|
|
|||
|
|
@ -16,10 +16,6 @@ import vts
|
|||
import preview
|
||||
import cv2
|
||||
|
||||
#overall, my changes seem to have memory usage up from about 140mb to 180mb
|
||||
#but cpu usage seems to be down
|
||||
#and it's more resilent to low performance scenarios
|
||||
|
||||
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument("-i", "--ip", help="Set IP address for sending tracking data", default="127.0.0.1")
|
||||
parser.add_argument("-p", "--port", type=int, help="Set port for sending tracking data", default=11573)
|
||||
|
|
@ -41,14 +37,6 @@ parser.add_argument("-v", "--visualize", type=int, help="Set this to 1 to visual
|
|||
|
||||
args = parser.parse_args()
|
||||
|
||||
os.environ["OMP_NUM_THREADS"] = str(args.landmark_detection_threads)
|
||||
|
||||
def messaging(messageQueue):
|
||||
while True:
|
||||
message = messageQueue.get()
|
||||
print(message)
|
||||
|
||||
|
||||
def visualize(frame, face, previewFrameQueue, face_center, face_radius):
|
||||
|
||||
y1 = int(face_center[1] - face_radius[1])
|
||||
|
|
@ -103,17 +91,10 @@ if fps > 30:
|
|||
#creating queues
|
||||
frameQueue = multiprocessing.Queue(maxsize=frameQueueSize)
|
||||
faceQueue = multiprocessing.Queue(maxsize=1)
|
||||
messageQueue = queue.Queue()
|
||||
packetQueue = queue.Queue()
|
||||
|
||||
#creating threads
|
||||
#probably doesn't need it's own thread, but it can't hurt
|
||||
messageThread = threading.Thread(target = messaging, args = [ messageQueue],)
|
||||
messageThread.daemon = True
|
||||
messageThread.start()
|
||||
|
||||
#I want to decouple the requests so they don't block anything else if they're slow
|
||||
VTS = vts.VTS(target_ip, target_port, silent, height, width, packetQueue, messageQueue)
|
||||
VTS = vts.VTS(target_ip, target_port, silent, height, width, packetQueue)
|
||||
packetSenderThread = threading.Thread(target = VTS.packetSender, args = [],)
|
||||
packetSenderThread.daemon = True
|
||||
packetSenderThread.start()
|
||||
|
|
@ -122,7 +103,7 @@ packetSenderThread.start()
|
|||
#not a very active process, it's blocked by the main thread 90% of the time
|
||||
#but putting it in it's own process means there's a little bit less occupying the same core as the main thread
|
||||
#which helps when the system is under load
|
||||
webcamProcess = multiprocessing.Process(target=webcam.startProcess,args = (frameQueue, faceQueue, fps, targetBrightness, width, height, mirror_input,messageQueue, ))
|
||||
webcamProcess = multiprocessing.Process(target=webcam.startProcess,args = (frameQueue, faceQueue, fps, targetBrightness, width, height, mirror_input, ))
|
||||
webcamProcess.daemon = True
|
||||
webcamProcess.start()
|
||||
|
||||
|
|
@ -143,7 +124,7 @@ peak_camera_latency = 0.0
|
|||
total_camera_latency = 0.0
|
||||
failures = 0
|
||||
|
||||
tracker = Tracker(width, height, messageQueue, featureType, threshold=args.threshold, silent=silent, model_type=args.model, detection_threshold=args.detection_threshold)
|
||||
tracker = Tracker(width, height, featureType, threshold=args.threshold, silent=silent, model_type=args.model, detection_threshold=args.detection_threshold)
|
||||
|
||||
#don't start until the webcam is ready
|
||||
while frameQueue.qsize() < 1:
|
||||
|
|
@ -170,14 +151,13 @@ try:
|
|||
#there's no reason to ever wait on this process or give it much of a queue
|
||||
if previewFrameQueue.qsize() < 1:
|
||||
previewFrameQueue.put(frame)
|
||||
|
||||
try:
|
||||
face, face_center, face_radius = tracker.predict(frame)
|
||||
failures = 0
|
||||
except Exception as e:
|
||||
if e.__class__ == KeyboardInterrupt:
|
||||
if not silent:
|
||||
messageQueue.put("Quitting")
|
||||
print("Quitting")
|
||||
break
|
||||
traceback.print_exc()
|
||||
failures += 1
|
||||
|
|
@ -186,14 +166,12 @@ try:
|
|||
|
||||
if face is not None:
|
||||
packet = VTS.preparePacket(face)
|
||||
#this can maybe cause weirdness in the gamma calculations
|
||||
#but I don't want to let that build a queue
|
||||
if faceQueue.qsize() < 1:
|
||||
faceQueue.put([face_center, face_radius])
|
||||
if visualizeFlag:
|
||||
visualize(frame, face, previewFrameQueue, face_center, face_radius)
|
||||
else:
|
||||
messageQueue.put("Camera behind, skipping frame")
|
||||
print("Camera behind, skipping frame")
|
||||
|
||||
duration = time.perf_counter() - frame_start
|
||||
|
||||
|
|
@ -203,7 +181,7 @@ try:
|
|||
time.sleep(target_duration - duration)
|
||||
else:
|
||||
#all caps because if this is consistently happening it's an issue
|
||||
messageQueue.put("CANNOT MAINTAIN FRAMERATE!")
|
||||
print("CANNOT MAINTAIN FRAMERATE!")
|
||||
|
||||
peak_time_between = max(peak_time_between, time.perf_counter() -frame_start)
|
||||
total_run_time += time.perf_counter() -frame_start
|
||||
|
|
@ -212,7 +190,7 @@ try:
|
|||
if(packet is not None):
|
||||
packetQueue.put(packet)
|
||||
else:
|
||||
messageQueue.put("No data sent to VTS")
|
||||
print("No data sent to VTS")
|
||||
|
||||
except KeyboardInterrupt:
|
||||
if not silent:
|
||||
|
|
|
|||
12
tracker.py
12
tracker.py
|
|
@ -21,8 +21,6 @@ def clamp_to_im(pt, w, h): #8 times per frame, but that only accounts for 0.005m
|
|||
y = h-1
|
||||
return (int(x),int(y+1))
|
||||
|
||||
|
||||
|
||||
class Models():
|
||||
def __init__(self, model_type=3):
|
||||
self.model_type = model_type
|
||||
|
|
@ -50,7 +48,7 @@ class Models():
|
|||
self.gazeTracker = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options, providers=providersList)
|
||||
|
||||
class Tracker():
|
||||
def __init__(self, width, height, messageQueue, featureType, model_type=3, detection_threshold=0.6, threshold=0.6, silent=False):
|
||||
def __init__(self, width, height, featureType, model_type=3, detection_threshold=0.6, threshold=0.6, silent=False):
|
||||
|
||||
self.detection_threshold = detection_threshold
|
||||
self.EyeTracker = eyes.EyeTracker()
|
||||
|
|
@ -66,7 +64,6 @@ class Tracker():
|
|||
self.threshold = threshold
|
||||
self.face = None
|
||||
self.face_info = face.FaceInfo(featureType)
|
||||
self.messageQueue = messageQueue
|
||||
|
||||
#scales cropped frame before sending to landmarks
|
||||
def preprocess(self, im):
|
||||
|
|
@ -103,10 +100,10 @@ class Tracker():
|
|||
return (crop, crop_info, duration_pp )
|
||||
|
||||
def early_exit(self, reason, start):
|
||||
self.messageQueue.put(reason)
|
||||
print(reason)
|
||||
self.face = None
|
||||
duration = (time.perf_counter() - start) * 1000
|
||||
self.messageQueue.put(f"Took {duration:.2f}ms")
|
||||
print(f"Took {duration:.2f}ms")
|
||||
return None, None, None
|
||||
|
||||
def predict(self, frame):
|
||||
|
|
@ -129,6 +126,7 @@ class Tracker():
|
|||
if crop is None:
|
||||
return self.early_exit("No valid crops", start)
|
||||
start_model = time.perf_counter()
|
||||
|
||||
confidence, lms = landmarks.run(self.model, crop, crop_info)
|
||||
#Early exit if below confidence threshold
|
||||
if confidence < self.threshold:
|
||||
|
|
@ -153,7 +151,7 @@ class Tracker():
|
|||
self.face = [y1, x1, y2 - y1, x2 - x1]
|
||||
duration_pnp += 1000 * (time.perf_counter() - start_pnp)
|
||||
duration = (time.perf_counter() - start) * 1000
|
||||
self.messageQueue.put(f"Took {duration:.2f}ms (detect: {duration_fd:.2f}ms, crop: {duration_pp:.2f}ms, track: {duration_model:.2f}ms, 3D points: {duration_pnp:.2f}ms)")
|
||||
print(f"Took {duration:.2f}ms (detect: {duration_fd:.2f}ms, crop: {duration_pp:.2f}ms, track: {duration_model:.2f}ms, 3D points: {duration_pnp:.2f}ms)")
|
||||
return face_info, self.EyeTracker.faceCenter, self.EyeTracker.faceRadius
|
||||
|
||||
#Combined multiple failures into one catch all exit
|
||||
|
|
|
|||
5
vts.py
5
vts.py
|
|
@ -3,7 +3,7 @@ import socket
|
|||
import struct
|
||||
|
||||
class VTS():
|
||||
def __init__(self, target_ip, target_port, silent, height, width, packetQueue, messageQueue):
|
||||
def __init__(self, target_ip, target_port, silent, height, width, packetQueue):
|
||||
|
||||
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
self.target_ip = target_ip
|
||||
|
|
@ -12,7 +12,6 @@ class VTS():
|
|||
self.features = ["eye_l", "eye_r", "eyebrow_steepness_l", "eyebrow_updown_l", "eyebrow_quirk_l", "eyebrow_steepness_r", "eyebrow_updown_r", "eyebrow_quirk_r", "mouth_corner_updown_l", "mouth_corner_inout_l", "mouth_corner_updown_r", "mouth_corner_inout_r", "mouth_open", "mouth_wide"]
|
||||
self.width = width
|
||||
self.height = height
|
||||
self.messageQueue = messageQueue
|
||||
self.packetQueue = packetQueue
|
||||
|
||||
def packetSender(self):
|
||||
|
|
@ -68,5 +67,5 @@ class VTS():
|
|||
try:
|
||||
self.sock.sendto(packet, (self.target_ip, self.target_port))
|
||||
except:
|
||||
self.messageQueue.put("Failed to send packet")
|
||||
print("Failed to send packet")
|
||||
return
|
||||
|
|
|
|||
|
|
@ -6,12 +6,12 @@ import math
|
|||
|
||||
|
||||
#this is just here for conveience
|
||||
def startProcess(frameQueue, faceQueue, fps, targetBrightness, width, height, mirrorInput, messageQueue):
|
||||
webcam = Webcam(frameQueue, faceQueue, fps, targetBrightness, width, height, mirrorInput, messageQueue)
|
||||
def startProcess(frameQueue, faceQueue, fps, targetBrightness, width, height, mirrorInput):
|
||||
webcam = Webcam(frameQueue, faceQueue, fps, targetBrightness, width, height, mirrorInput)
|
||||
webcam.start()
|
||||
|
||||
class Webcam():
|
||||
def __init__(self, frameQueue, faceQueue, fps, targetBrightness, width, height, mirrorInput, messageQueue):
|
||||
def __init__(self, frameQueue, faceQueue, fps, targetBrightness, width, height, mirrorInput):
|
||||
self.cap = cv2.VideoCapture(0)
|
||||
#tbh, these settings will make things way slower and frametimes will suffer
|
||||
#but they're the only way frames bigger than 480p will work
|
||||
|
|
@ -31,7 +31,6 @@ class Webcam():
|
|||
self.cameraLatency = 0
|
||||
self.brightnessFrame = None #a copy of the brightness channel for gamma calculations
|
||||
self.targetBrightness = targetBrightness #the target average brightness of the face, used in gamma calculations
|
||||
self.messageQueue = messageQueue #outgoing queue, unused, but I wanted it here in case I wanted it
|
||||
self.width = width #unused, but it seemed useful to have around
|
||||
self.height = height #unused, but it seemed useful to have around
|
||||
self.frame = None
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue