copilot/OpenSeeFace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

stuff

Browse source
This commit is contained in:
Cowwan 2024-02-09 20:39:51 -05:00
parent dbf5f6ab78
commit f288689960
11 changed files with 175 additions and 146 deletions
Download patch file Download diff file Expand all files Collapse all files

BIN
.stats.kate-swp Normal file
View file

Binary file not shown.

11
eyes.py
View file

@ -1,6 +1,7 @@
import numpy as np import numpy as np
import math import math
import cv2 import cv2
cv2.setNumThreads(6)
def clamp_to_im(pt, w, h): def clamp_to_im(pt, w, h):
x = pt[0] x = pt[0]
@ -33,14 +34,14 @@ def rotate_image(image, a, center): #twice per frame, 0.2ms - 0.25ms each, impro
return image return image
class Eye(): class Eye():
mean = np.float32(np.array([-2.1179, -2.0357, -1.8044]))
std = np.float32(np.array([0.0171, 0.0175, 0.0174]))
def __init__(self,index): def __init__(self,index):
self.eye_tracking_frames = 0 self.eye_tracking_frames = 0
self.index = index self.index = index
self.state = [1.,0.,0.,0.] self.state = [1.,0.,0.,0.]
self.image = None self.image = None
self.info = None self.info = None
self.mean = np.float32(np.array([-2.1179, -2.0357, -1.8044]))
self.std = np.float32(np.array([0.0171, 0.0175, 0.0174]))
self.results = None self.results = None
self.condifence = 0 self.condifence = 0
self.standardDeviation = 0. self.standardDeviation = 0.
@ -144,12 +145,13 @@ class Eye():
self.standardDeviation = math.sqrt(self.averageEyeConfidenceVariance) self.standardDeviation = math.sqrt(self.averageEyeConfidenceVariance)
class EyeTracker(): class EyeTracker():
np.float32(np.array([-2.1179, -2.0357, -1.8044]))
std = np.float32(np.array([0.0171, 0.0175, 0.0174]))
def __init__(self): def __init__(self):
self.leftEye = Eye(1) self.leftEye = Eye(1)
self.rightEye = Eye(0) self.rightEye = Eye(0)
self.mean = np.float32(np.array([-2.1179, -2.0357, -1.8044]))
self.std = np.float32(np.array([0.0171, 0.0175, 0.0174]))
self.offset = None self.offset = None
self.face = None
self.faceCenter = None self.faceCenter = None
self.faceRadius = None self.faceRadius = None
@ -187,5 +189,6 @@ class EyeTracker():
x1, y1 = clamp_to_im(self.faceCenter - self.faceRadius , h, w) x1, y1 = clamp_to_im(self.faceCenter - self.faceRadius , h, w)
x2, y2 = clamp_to_im(self.faceCenter + self.faceRadius + 1, h, w) x2, y2 = clamp_to_im(self.faceCenter + self.faceRadius + 1, h, w)
self.offset = np.array((x1, y1)) self.offset = np.array((x1, y1))
self.face = [x1, y1, x2 - x1, y2 - y1]
lms = (lms[:, 0:2] - self.offset).astype(int) lms = (lms[:, 0:2] - self.offset).astype(int)
return lms, frame[y1:y2, x1:x2] return lms, frame[y1:y2, x1:x2]

147
face.py
View file

@ -4,81 +4,82 @@ import featureExtractor
import emilianaFeatureExtractor import emilianaFeatureExtractor
class FaceInfo(): class FaceInfo():
face_3d = np.array([
[ 0.4551769692672 , 0.300895790030204, -0.764429433974752],
[ 0.448998827123556, 0.166995837790733, -0.765143004071253],
[ 0.437431554952677, 0.022655479179981, -0.739267175112735],
[ 0.415033422928434, -0.088941454648772, -0.747947437846473],
[ 0.389123587370091, -0.232380029794684, -0.704788385327458],
[ 0.334630113904382, -0.361265387599081, -0.615587579236862],
[ 0.263725112132858, -0.460009725616771, -0.491479221041573],
[ 0.16241621322721 , -0.558037146073869, -0.339445180872282],
[ 0. , -0.621079019321682, -0.287294770748887],
[-0.16241621322721 , -0.558037146073869, -0.339445180872282],
[-0.263725112132858, -0.460009725616771, -0.491479221041573],
[-0.334630113904382, -0.361265387599081, -0.615587579236862],
[-0.389123587370091, -0.232380029794684, -0.704788385327458],
[-0.415033422928434, -0.088941454648772, -0.747947437846473],
[-0.437431554952677, 0.022655479179981, -0.739267175112735],
[-0.448998827123556, 0.166995837790733, -0.765143004071253],
[-0.4551769692672 , 0.300895790030204, -0.764429433974752],
[ 0.385529968662985, 0.402800553948697, -0.310031082540741],
[ 0.322196658344302, 0.464439136821772, -0.250558059367669],
[ 0.25409760441282 , 0.46420381416882 , -0.208177722146526],
[ 0.186875436782135, 0.44706071961879 , -0.145299823706503],
[ 0.120880983543622, 0.423566314072968, -0.110757158774771],
[-0.120880983543622, 0.423566314072968, -0.110757158774771],
[-0.186875436782135, 0.44706071961879 , -0.145299823706503],
[-0.25409760441282 , 0.46420381416882 , -0.208177722146526],
[-0.322196658344302, 0.464439136821772, -0.250558059367669],
[-0.385529968662985, 0.402800553948697, -0.310031082540741],
[ 0. , 0.293332603215811, -0.137582088779393],
[ 0. , 0.194828701837823, -0.069158109325951],
[ 0. , 0.103844017393155, -0.009151819844964],
[ 0. , 0. , 0. ],
[ 0.080626352317973, -0.041276068128093, -0.134161035564826],
[ 0.046439347377934, -0.057675223874769, -0.102990627164664],
[ 0. , -0.068753126205604, -0.090545348482397],
[-0.046439347377934, -0.057675223874769, -0.102990627164664],
[-0.080626352317973, -0.041276068128093, -0.134161035564826],
[ 0.315905195966084, 0.298337502555443, -0.285107407636464],
[ 0.275252345439353, 0.312721904921771, -0.244558251170671],
[ 0.176394511553111, 0.311907184376107, -0.219205360345231],
[ 0.131229723798772, 0.284447361805627, -0.234239149487417],
[ 0.184124948330084, 0.260179585304867, -0.226590776513707],
[ 0.279433549294448, 0.267363071770222, -0.248441437111633],
[-0.131229723798772, 0.284447361805627, -0.234239149487417],
[-0.176394511553111, 0.311907184376107, -0.219205360345231],
[-0.275252345439353, 0.312721904921771, -0.244558251170671],
[-0.315905195966084, 0.298337502555443, -0.285107407636464],
[-0.279433549294448, 0.267363071770222, -0.248441437111633],
[-0.184124948330084, 0.260179585304867, -0.226590776513707],
[ 0.121155252430729, -0.208988660580347, -0.160606287940521],
[ 0.041356305910044, -0.194484199722098, -0.096159882202821],
[ 0. , -0.205180167345702, -0.083299217789729],
[-0.041356305910044, -0.194484199722098, -0.096159882202821],
[-0.121155252430729, -0.208988660580347, -0.160606287940521],
[-0.132325402795928, -0.290857984604968, -0.187067868218105],
[-0.064137791831655, -0.325377847425684, -0.158924039726607],
[ 0. , -0.343742581679188, -0.113925986025684],
[ 0.064137791831655, -0.325377847425684, -0.158924039726607],
[ 0.132325402795928, -0.290857984604968, -0.187067868218105],
[ 0.181481567104525, -0.243239316141725, -0.231284988892766],
[ 0.083999507750469, -0.239717753728704, -0.155256465640701],
[ 0. , -0.256058040176369, -0.0950619498899 ],
[-0.083999507750469, -0.239717753728704, -0.155256465640701],
[-0.181481567104525, -0.243239316141725, -0.231284988892766],
[-0.074036069749345, -0.250689938345682, -0.177346470406188],
[ 0. , -0.264945854681568, -0.112349967428413],
[ 0.074036069749345, -0.250689938345682, -0.177346470406188],
# Pupils and eyeball centers
[ 0.257990002632141, 0.276080012321472, -0.219998998939991],
[-0.257990002632141, 0.276080012321472, -0.219998998939991],
[ 0.257990002632141, 0.276080012321472, -0.324570998549461],
[-0.257990002632141, 0.276080012321472, -0.324570998549461]
], np.float32)
def __init__(self, featureType): def __init__(self, featureType):
self.featureType = featureType self.featureType = featureType
self.face_3d = np.array([
[ 0.4551769692672 , 0.300895790030204, -0.764429433974752],
[ 0.448998827123556, 0.166995837790733, -0.765143004071253],
[ 0.437431554952677, 0.022655479179981, -0.739267175112735],
[ 0.415033422928434, -0.088941454648772, -0.747947437846473],
[ 0.389123587370091, -0.232380029794684, -0.704788385327458],
[ 0.334630113904382, -0.361265387599081, -0.615587579236862],
[ 0.263725112132858, -0.460009725616771, -0.491479221041573],
[ 0.16241621322721 , -0.558037146073869, -0.339445180872282],
[ 0. , -0.621079019321682, -0.287294770748887],
[-0.16241621322721 , -0.558037146073869, -0.339445180872282],
[-0.263725112132858, -0.460009725616771, -0.491479221041573],
[-0.334630113904382, -0.361265387599081, -0.615587579236862],
[-0.389123587370091, -0.232380029794684, -0.704788385327458],
[-0.415033422928434, -0.088941454648772, -0.747947437846473],
[-0.437431554952677, 0.022655479179981, -0.739267175112735],
[-0.448998827123556, 0.166995837790733, -0.765143004071253],
[-0.4551769692672 , 0.300895790030204, -0.764429433974752],
[ 0.385529968662985, 0.402800553948697, -0.310031082540741],
[ 0.322196658344302, 0.464439136821772, -0.250558059367669],
[ 0.25409760441282 , 0.46420381416882 , -0.208177722146526],
[ 0.186875436782135, 0.44706071961879 , -0.145299823706503],
[ 0.120880983543622, 0.423566314072968, -0.110757158774771],
[-0.120880983543622, 0.423566314072968, -0.110757158774771],
[-0.186875436782135, 0.44706071961879 , -0.145299823706503],
[-0.25409760441282 , 0.46420381416882 , -0.208177722146526],
[-0.322196658344302, 0.464439136821772, -0.250558059367669],
[-0.385529968662985, 0.402800553948697, -0.310031082540741],
[ 0. , 0.293332603215811, -0.137582088779393],
[ 0. , 0.194828701837823, -0.069158109325951],
[ 0. , 0.103844017393155, -0.009151819844964],
[ 0. , 0. , 0. ],
[ 0.080626352317973, -0.041276068128093, -0.134161035564826],
[ 0.046439347377934, -0.057675223874769, -0.102990627164664],
[ 0. , -0.068753126205604, -0.090545348482397],
[-0.046439347377934, -0.057675223874769, -0.102990627164664],
[-0.080626352317973, -0.041276068128093, -0.134161035564826],
[ 0.315905195966084, 0.298337502555443, -0.285107407636464],
[ 0.275252345439353, 0.312721904921771, -0.244558251170671],
[ 0.176394511553111, 0.311907184376107, -0.219205360345231],
[ 0.131229723798772, 0.284447361805627, -0.234239149487417],
[ 0.184124948330084, 0.260179585304867, -0.226590776513707],
[ 0.279433549294448, 0.267363071770222, -0.248441437111633],
[-0.131229723798772, 0.284447361805627, -0.234239149487417],
[-0.176394511553111, 0.311907184376107, -0.219205360345231],
[-0.275252345439353, 0.312721904921771, -0.244558251170671],
[-0.315905195966084, 0.298337502555443, -0.285107407636464],
[-0.279433549294448, 0.267363071770222, -0.248441437111633],
[-0.184124948330084, 0.260179585304867, -0.226590776513707],
[ 0.121155252430729, -0.208988660580347, -0.160606287940521],
[ 0.041356305910044, -0.194484199722098, -0.096159882202821],
[ 0. , -0.205180167345702, -0.083299217789729],
[-0.041356305910044, -0.194484199722098, -0.096159882202821],
[-0.121155252430729, -0.208988660580347, -0.160606287940521],
[-0.132325402795928, -0.290857984604968, -0.187067868218105],
[-0.064137791831655, -0.325377847425684, -0.158924039726607],
[ 0. , -0.343742581679188, -0.113925986025684],
[ 0.064137791831655, -0.325377847425684, -0.158924039726607],
[ 0.132325402795928, -0.290857984604968, -0.187067868218105],
[ 0.181481567104525, -0.243239316141725, -0.231284988892766],
[ 0.083999507750469, -0.239717753728704, -0.155256465640701],
[ 0. , -0.256058040176369, -0.0950619498899 ],
[-0.083999507750469, -0.239717753728704, -0.155256465640701],
[-0.181481567104525, -0.243239316141725, -0.231284988892766],
[-0.074036069749345, -0.250689938345682, -0.177346470406188],
[ 0. , -0.264945854681568, -0.112349967428413],
[ 0.074036069749345, -0.250689938345682, -0.177346470406188],
# Pupils and eyeball centers
[ 0.257990002632141, 0.276080012321472, -0.219998998939991],
[-0.257990002632141, 0.276080012321472, -0.219998998939991],
[ 0.257990002632141, 0.276080012321472, -0.324570998549461],
[-0.257990002632141, 0.276080012321472, -0.324570998549461]
], np.float32)
self.reset() self.reset()

39
facedetection.py
View file

@ -1,34 +1,35 @@
import numpy as np import numpy as np
import cv2 import cv2
cv2.setNumThreads(6)
def detect_faces(frame, model, detection_threshold ): def detect_faces(frame, model, detection_threshold ):
image = resizeImage(frame) image = resizeImage(frame)
outputs, maxpool = model.faceDetection.run([], {'input': image}) outputs, maxpool = model.faceDetection.run([], {'input': image})
outputs = outputs[0] outputs = outputs[0]
maxpool = maxpool[0] maxpool = maxpool[0]
mask = outputs[0] == maxpool[0] mask = outputs[0] == maxpool[0]
mask2 = outputs[1] > 0.15 mask2 = outputs[1] > 0.15
outputs[0] = outputs[0] * mask.astype(int) * mask2.astype(int) outputs[0] = outputs[0] * mask.astype(int) * mask2.astype(int)
faceLocation = np.argmax(outputs[0].flatten()) faceLocation = np.argmax(outputs[0].flatten())
x = faceLocation % 56 x = faceLocation % 56
y = faceLocation // 56 y = faceLocation // 56
if outputs[0, y, x] < detection_threshold: if outputs[0, y, x] < detection_threshold:
return None return None
r = outputs[1, y, x] * 112. r = outputs[1, y, x] * 112.
results= (((x * 4) - r, (y * 4) - r, r*2,r*2)) results= (((x * 4) - r, (y * 4) - r, r*2,r*2))
results = np.array(results).astype(np.float32) results = np.array(results).astype(np.float32)
results[[0,2]] *= frame.shape[1] / 224. results[[0,2]] *= frame.shape[1] / 224.
results[[1,3]] *= frame.shape[0] / 224. results[[1,3]] *= frame.shape[0] / 224.
return results return results
def resizeImage(frame): def resizeImage(frame):
mean = np.float32(np.array([-2.1179, -2.0357, -1.8044])) mean = np.float32(np.array([-2.1179, -2.0357, -1.8044]))
std = np.float32(np.array([0.0171, 0.0175, 0.0174])) std = np.float32(np.array([0.0171, 0.0175, 0.0174]))
targetimageSize = [224, 224] targetimageSize = [224, 224]
image = cv2.resize(frame, targetimageSize, interpolation=cv2.INTER_LINEAR) * std + mean image = cv2.resize(frame, targetimageSize, interpolation=cv2.INTER_CUBIC) * std + mean
image = np.expand_dims(image, 0) image = np.expand_dims(image, 0)
image = np.transpose(image, (0,3,1,2)) image = np.transpose(image, (0,3,1,2))
return image return image

30
facetracker.py
View file

@ -26,14 +26,13 @@ parser.add_argument("--model", type=int, help="This can be used to select the tr
parser.add_argument("--preview", type=int, help="Preview the frames sent to the tracker", default=0) parser.add_argument("--preview", type=int, help="Preview the frames sent to the tracker", default=0)
parser.add_argument("--feature-type", type=int, help="Sets which version of feature extraction is used. 0 is my new version that works well for me and allows for some customization, 1 is EmilianaVT's version aka, normal OpenSeeFace operation", default=0, choices=[0, 1]) parser.add_argument("--feature-type", type=int, help="Sets which version of feature extraction is used. 0 is my new version that works well for me and allows for some customization, 1 is EmilianaVT's version aka, normal OpenSeeFace operation", default=0, choices=[0, 1])
parser.add_argument("--numpy-threads", type=int, help="Numer of threads Numpy can use, doesn't seem to effect much", default=1) parser.add_argument("--numpy-threads", type=int, help="Numer of threads Numpy can use, doesn't seem to effect much", default=1)
parser.add_argument("-T","--threads", type=int, help="Numer of threads used for landmark detection. Default is 1 (~15ms per frame on my computer), 2 gets slightly faster frames (~10ms on my computer), more than 2 doesn't seem to help much", default=1) parser.add_argument("-T","--threads", type=int, help="Numer of threads used for landmark detection. Default is 1 (~15ms per frame on my computer), 2 gets slightly faster frames (~10ms on my computer), more than 2 doesn't seem to help much", default=2)
parser.add_argument("-v", "--visualize", type=int, help="Set this to 1 to visualize the tracking", default=0) parser.add_argument("-v", "--visualize", type=int, help="Set this to 1 to visualize the tracking", default=0)
args = parser.parse_args() args = parser.parse_args()
#processing args #processing args
#For Vtube Studio: ip 127.0.0.1 port 11573
target_ip = args.ip target_ip = args.ip
target_port = args.port target_port = args.port
fps = args.fps fps = args.fps
@ -53,7 +52,8 @@ silent = (args.silent == 1)
if height > 480: if height > 480:
frameQueueSize = 2 frameQueueSize = 2
else: else:
frameQueueSize = 1 frameQueueSize = 2
frameQueue = multiprocessing.Queue(maxsize=frameQueueSize) frameQueue = multiprocessing.Queue(maxsize=frameQueueSize)
faceQueue = multiprocessing.Queue(maxsize=1) faceQueue = multiprocessing.Queue(maxsize=1)
packetQueue = queue.Queue() packetQueue = queue.Queue()
@ -65,9 +65,7 @@ packetSenderThread.daemon = True
packetSenderThread.start() packetSenderThread.start()
#creating processes #creating processes
#not a very active process, it's blocked by the main thread 90% of the time
#but putting it in it's own process means there's a little bit less occupying the same core as the main thread
#which helps when the system is under load
webcamProcess = multiprocessing.Process(target=webcam.startProcess,args = (frameQueue, faceQueue, fps, targetBrightness, width, height, mirror_input, )) webcamProcess = multiprocessing.Process(target=webcam.startProcess,args = (frameQueue, faceQueue, fps, targetBrightness, width, height, mirror_input, ))
webcamProcess.daemon = True webcamProcess.daemon = True
webcamProcess.start() webcamProcess.start()
@ -119,23 +117,23 @@ try:
if previewFrameQueue.qsize() < 1: if previewFrameQueue.qsize() < 1:
previewFrameQueue.put(frame) previewFrameQueue.put(frame)
face, face_center, face_radius = tracker.predict(frame) faceInfo, face = tracker.predict(frame)
if face is not None: if faceInfo is not None:
packet = VTS.preparePacket(face) packet = VTS.preparePacket(faceInfo)
if faceQueue.qsize() < 1: if faceQueue.qsize() < 1:
faceQueue.put([face_center, face_radius]) faceQueue.put(face)
if visualizeFlag: if visualizeFlag:
preview.visualize(frame, face, previewFrameQueue, face_center, face_radius) preview.visualize(frame, faceInfo, previewFrameQueue, face_center, face_radius)
duration = time.perf_counter() - frame_get duration = time.perf_counter() - frame_get
total_active_time += duration total_active_time += duration
peak_frame_time = max(peak_frame_time, duration) peak_frame_time = max(peak_frame_time, duration)
if duration < target_duration:
time.sleep(target_duration - duration)
sleepTimer = max(time.perf_counter() - frame_get, sleepTimer) else:
time.sleep(max(peak_frame_time - duration, 0)) #all caps because if this is consistently happening it's an issue
sleepTimer -= 0.0005 messageQueue.put("CANNOT MAINTAIN FRAMERATE!")
peak_time_between = max(peak_time_between, time.perf_counter() -frame_start) peak_time_between = max(peak_time_between, time.perf_counter() -frame_start)
total_run_time += time.perf_counter() -frame_start total_run_time += time.perf_counter() -frame_start

0
image.py Normal file
View file

1
landmarks.py
View file

@ -1,6 +1,7 @@
import numpy as np import numpy as np
import math import math
import cv2 import cv2
cv2.setNumThreads(6)
def matrix_to_quaternion(m): def matrix_to_quaternion(m):
t = 0.0 t = 0.0

1
preview.py
View file

@ -1,5 +1,6 @@
import cv2 import cv2
import multiprocessing import multiprocessing
cv2.setNumThreads(6)
#this is it's own file just for ceanlieness #this is it's own file just for ceanlieness

54
tracker.py
View file

@ -2,6 +2,7 @@ import os
import numpy as np import numpy as np
import onnxruntime import onnxruntime
import cv2 import cv2
cv2.setNumThreads(6)
import time import time
import facedetection import facedetection
import eyes import eyes
@ -9,27 +10,34 @@ import landmarks
import face import face
#this file is so much smaller than it used to be, lol #this file is so much smaller than it used to be, lol
#I moved most of the functionality into separate files so they're easier to work with #I moved most of the functionality into separate files so they're easier to work with
def clamp_to_im(pt, w, h): #8 times per frame, but that only accounts for 0.005ms def clamp_to_im(pt, w, h):
x=max(pt[0],0) x = pt[0]
y=max(pt[1],0) y = pt[1]
if x < 0:
x = 0
if y < 0:
y = 0
if x >= w: if x >= w:
x = w-1 x = w-1
if y >= h: if y >= h:
y = h-1 y = h-1
return (int(x),int(y+1)) return (int(x), int(y+1))
class Models(): class Models():
models = [
"lm_model0_opt.onnx",
"lm_model1_opt.onnx",
"lm_model2_opt.onnx",
"lm_model3_opt.onnx",
"lm_model4_opt.onnx"]
def __init__(self, threads, model_type=3): def __init__(self, threads, model_type=3):
self.model_type = model_type self.model_type = model_type
self.models = [
"lm_model0_opt.onnx",
"lm_model1_opt.onnx",
"lm_model2_opt.onnx",
"lm_model3_opt.onnx",
"lm_model4_opt.onnx"]
model = self.models[self.model_type] model = self.models[self.model_type]
model_base_path = os.path.join(os.path.dirname(__file__), os.path.join("models")) model_base_path = os.path.join(os.path.dirname(__file__), os.path.join("models"))
@ -48,6 +56,9 @@ class Models():
self.gazeTracker = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options, providers=providersList) self.gazeTracker = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options, providers=providersList)
class Tracker(): class Tracker():
targetimageSize = [224, 224]
std = np.float32(np.array([0.0171, 0.0175, 0.0174]))
mean = np.float32(np.array([-2.1179, -2.0357, -1.8044]))
def __init__(self, width, height, featureType, threads, model_type=3, detection_threshold=0.6, threshold=0.6, silent=False): def __init__(self, width, height, featureType, threads, model_type=3, detection_threshold=0.6, threshold=0.6, silent=False):
self.detection_threshold = detection_threshold self.detection_threshold = detection_threshold
@ -55,9 +66,6 @@ class Tracker():
self.model = Models(threads, model_type = model_type) self.model = Models(threads, model_type = model_type)
# Image normalization constants # Image normalization constants
self.mean = np.float32(np.array([-2.1179, -2.0357, -1.8044]))
self.std = np.float32(np.array([0.0171, 0.0175, 0.0174]))
self.targetimageSize = [224, 224]
self.width = width self.width = width
self.height = height self.height = height
@ -75,12 +83,10 @@ class Tracker():
duration_pp = 0.0 duration_pp = 0.0
x,y,w,h = self.face x,y,w,h = self.face
wint = int(w * 0.1) crop_x1 = x - (w * 0.1)
hint = int(h * 0.125) crop_y1 = y - (h * 0.125)
crop_x1 = x - wint crop_x2 = x + w + (w * 0.1)
crop_y1 = y - hint crop_y2 = y + h + (h * 0.125)
crop_x2 = x + w + wint
crop_y2 = y + h + hint
crop_x1, crop_y1 = clamp_to_im((crop_x1, crop_y1), self.width, self.height) crop_x1, crop_y1 = clamp_to_im((crop_x1, crop_y1), self.width, self.height)
crop_x2, crop_y2 = clamp_to_im((crop_x2, crop_y2), self.width, self.height) crop_x2, crop_y2 = clamp_to_im((crop_x2, crop_y2), self.width, self.height)
@ -104,7 +110,7 @@ class Tracker():
self.face = None self.face = None
duration = (time.perf_counter() - start) * 1000 duration = (time.perf_counter() - start) * 1000
print(f"Took {duration:.2f}ms") print(f"Took {duration:.2f}ms")
return None, None, None return None, None
def predict(self, frame): def predict(self, frame):
start = time.perf_counter() start = time.perf_counter()
@ -146,13 +152,13 @@ class Tracker():
if face_info.success: if face_info.success:
face_info.adjust_3d() face_info.adjust_3d()
lms = face_info.lms[:, 0:2] lms = face_info.lms[:, 0:2]
x1, y1 = lms[0:66].min(0) y1, x1 = lms[0:66].min(0)
x2, y2 = lms[0:66].max(0) y2, x2 = lms[0:66].max(0)
self.face = [y1, x1, y2 - y1, x2 - x1] self.face = [x1, y1, x2 - x1, y2 - y1]
duration_pnp += 1000 * (time.perf_counter() - start_pnp) duration_pnp += 1000 * (time.perf_counter() - start_pnp)
duration = (time.perf_counter() - start) * 1000 duration = (time.perf_counter() - start) * 1000
print(f"Took {duration:.2f}ms (detect: {duration_fd:.2f}ms, crop: {duration_pp:.2f}ms, track: {duration_model:.2f}ms, 3D points: {duration_pnp:.2f}ms)") print(f"Took {duration:.2f}ms (detect: {duration_fd:.2f}ms, crop: {duration_pp:.2f}ms, track: {duration_model:.2f}ms, 3D points: {duration_pnp:.2f}ms)")
return face_info, self.EyeTracker.faceCenter, self.EyeTracker.faceRadius return face_info, self.face
#Combined multiple failures into one catch all exit #Combined multiple failures into one catch all exit
return self.early_exit("Face info not valid", start) return self.early_exit("Face info not valid", start)

3
vts.py
View file

@ -3,13 +3,14 @@ import socket
import struct import struct
class VTS(): class VTS():
features = ["eye_l", "eye_r", "eyebrow_steepness_l", "eyebrow_updown_l", "eyebrow_quirk_l", "eyebrow_steepness_r", "eyebrow_updown_r", "eyebrow_quirk_r", "mouth_corner_updown_l", "mouth_corner_inout_l", "mouth_corner_updown_r", "mouth_corner_inout_r", "mouth_open", "mouth_wide"]
def __init__(self, target_ip, target_port, silent, height, width, packetQueue): def __init__(self, target_ip, target_port, silent, height, width, packetQueue):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.target_ip = target_ip self.target_ip = target_ip
self.target_port = target_port self.target_port = target_port
self.silent = silent self.silent = silent
self.features = ["eye_l", "eye_r", "eyebrow_steepness_l", "eyebrow_updown_l", "eyebrow_quirk_l", "eyebrow_steepness_r", "eyebrow_updown_r", "eyebrow_quirk_r", "mouth_corner_updown_l", "mouth_corner_inout_l", "mouth_corner_updown_r", "mouth_corner_inout_r", "mouth_open", "mouth_wide"]
self.width = width self.width = width
self.height = height self.height = height
self.packetQueue = packetQueue self.packetQueue = packetQueue

35
webcam.py
View file

@ -3,11 +3,24 @@ import os
os.environ["OMP_NUM_THREADS"] = str(1) os.environ["OMP_NUM_THREADS"] = str(1)
import dshowcapture import dshowcapture
import cv2 import cv2
cv2.setNumThreads(6)
import multiprocessing import multiprocessing
import time import time
import numpy as np import numpy as np
import math import math
def clamp_to_im(pt, w, h):
x = pt[0]
y = pt[1]
if x < 0:
x = 0
if y < 0:
y = 0
if x >= w:
x = w-1
if y >= h:
y = h-1
return (int(x), int(y+1))
#this is just here for conveience #this is just here for conveience
def startProcess(frameQueue, faceQueue, fps, targetBrightness, width, height, mirrorInput): def startProcess(frameQueue, faceQueue, fps, targetBrightness, width, height, mirrorInput):
@ -43,6 +56,7 @@ class Webcam():
self.frame = None self.frame = None
def start(self): def start(self):
while self.cap.isOpened(): while self.cap.isOpened():
frameStart= time.perf_counter() frameStart= time.perf_counter()
totalFrameLatency = time.perf_counter() totalFrameLatency = time.perf_counter()
@ -91,17 +105,20 @@ class Webcam():
#calculate the ideal gamma based on the brightness of the user's face #calculate the ideal gamma based on the brightness of the user's face
#kind of winging it with the math, but it's a lot better than calculating based on the whole frame #kind of winging it with the math, but it's a lot better than calculating based on the whole frame
def updateGamma(self): def updateGamma(self):
face_center, face_radius = self.faceQueue.get() face = self.faceQueue.get()
y1 = int(face_center[1] - face_radius[1]) x,y,w,h = face
y2 = int(face_center[1] + face_radius[1]) crop_x1 = x
x1 = int(face_center[0] - face_radius[0]) crop_y1 = y
x2 = int(face_center[0] + face_radius[0]) crop_x2 = x + w
crop_y2 = y + h
#I was doing random sampling of the image brightness crop_x1, crop_y1 = clamp_to_im((crop_x1, crop_y1), self.width, self.height)
#and that's probably brighter for the whole frame crop_x2, crop_y2 = clamp_to_im((crop_x2, crop_y2), self.width, self.height)
#but for just the face this is pretty fast
averageBrightness = np.mean(self.brightnessFrame[y1:y2,x1:x2])/256 croppedFace = self.brightnessFrame[crop_y1:crop_y2, crop_x1:crop_x2]
averageBrightness = np.mean(croppedFace)/256
self.gamma = math.log(self.targetBrightness, averageBrightness) self.gamma = math.log(self.targetBrightness, averageBrightness)
return return