stuff
This commit is contained in:
Cowwan
parent
dbf5f6ab78
commit
f288689960
11 changed files with 175 additions and 146 deletions
BIN
.stats.kate-swp
Normal file
BIN
.stats.kate-swp
Normal file
Binary file not shown.
11
eyes.py
11
eyes.py
|
|
@ -1,6 +1,7 @@
|
|||
import numpy as np
|
||||
import math
|
||||
import cv2
|
||||
cv2.setNumThreads(6)
|
||||
|
||||
def clamp_to_im(pt, w, h):
|
||||
x = pt[0]
|
||||
|
|
@ -33,14 +34,14 @@ def rotate_image(image, a, center): #twice per frame, 0.2ms - 0.25ms each, impro
|
|||
return image
|
||||
|
||||
class Eye():
|
||||
mean = np.float32(np.array([-2.1179, -2.0357, -1.8044]))
|
||||
std = np.float32(np.array([0.0171, 0.0175, 0.0174]))
|
||||
def __init__(self,index):
|
||||
self.eye_tracking_frames = 0
|
||||
self.index = index
|
||||
self.state = [1.,0.,0.,0.]
|
||||
self.image = None
|
||||
self.info = None
|
||||
self.mean = np.float32(np.array([-2.1179, -2.0357, -1.8044]))
|
||||
self.std = np.float32(np.array([0.0171, 0.0175, 0.0174]))
|
||||
self.results = None
|
||||
self.condifence = 0
|
||||
self.standardDeviation = 0.
|
||||
|
|
@ -144,12 +145,13 @@ class Eye():
|
|||
self.standardDeviation = math.sqrt(self.averageEyeConfidenceVariance)
|
||||
|
||||
class EyeTracker():
|
||||
np.float32(np.array([-2.1179, -2.0357, -1.8044]))
|
||||
std = np.float32(np.array([0.0171, 0.0175, 0.0174]))
|
||||
def __init__(self):
|
||||
self.leftEye = Eye(1)
|
||||
self.rightEye = Eye(0)
|
||||
self.mean = np.float32(np.array([-2.1179, -2.0357, -1.8044]))
|
||||
self.std = np.float32(np.array([0.0171, 0.0175, 0.0174]))
|
||||
self.offset = None
|
||||
self.face = None
|
||||
self.faceCenter = None
|
||||
self.faceRadius = None
|
||||
|
||||
|
|
@ -187,5 +189,6 @@ class EyeTracker():
|
|||
x1, y1 = clamp_to_im(self.faceCenter - self.faceRadius , h, w)
|
||||
x2, y2 = clamp_to_im(self.faceCenter + self.faceRadius + 1, h, w)
|
||||
self.offset = np.array((x1, y1))
|
||||
self.face = [x1, y1, x2 - x1, y2 - y1]
|
||||
lms = (lms[:, 0:2] - self.offset).astype(int)
|
||||
return lms, frame[y1:y2, x1:x2]
|
||||
|
|
|
|||
147
face.py
147
face.py
|
|
@ -4,81 +4,82 @@ import featureExtractor
|
|||
import emilianaFeatureExtractor
|
||||
|
||||
class FaceInfo():
|
||||
face_3d = np.array([
|
||||
[ 0.4551769692672 , 0.300895790030204, -0.764429433974752],
|
||||
[ 0.448998827123556, 0.166995837790733, -0.765143004071253],
|
||||
[ 0.437431554952677, 0.022655479179981, -0.739267175112735],
|
||||
[ 0.415033422928434, -0.088941454648772, -0.747947437846473],
|
||||
[ 0.389123587370091, -0.232380029794684, -0.704788385327458],
|
||||
[ 0.334630113904382, -0.361265387599081, -0.615587579236862],
|
||||
[ 0.263725112132858, -0.460009725616771, -0.491479221041573],
|
||||
[ 0.16241621322721 , -0.558037146073869, -0.339445180872282],
|
||||
[ 0. , -0.621079019321682, -0.287294770748887],
|
||||
[-0.16241621322721 , -0.558037146073869, -0.339445180872282],
|
||||
[-0.263725112132858, -0.460009725616771, -0.491479221041573],
|
||||
[-0.334630113904382, -0.361265387599081, -0.615587579236862],
|
||||
[-0.389123587370091, -0.232380029794684, -0.704788385327458],
|
||||
[-0.415033422928434, -0.088941454648772, -0.747947437846473],
|
||||
[-0.437431554952677, 0.022655479179981, -0.739267175112735],
|
||||
[-0.448998827123556, 0.166995837790733, -0.765143004071253],
|
||||
[-0.4551769692672 , 0.300895790030204, -0.764429433974752],
|
||||
[ 0.385529968662985, 0.402800553948697, -0.310031082540741],
|
||||
[ 0.322196658344302, 0.464439136821772, -0.250558059367669],
|
||||
[ 0.25409760441282 , 0.46420381416882 , -0.208177722146526],
|
||||
[ 0.186875436782135, 0.44706071961879 , -0.145299823706503],
|
||||
[ 0.120880983543622, 0.423566314072968, -0.110757158774771],
|
||||
[-0.120880983543622, 0.423566314072968, -0.110757158774771],
|
||||
[-0.186875436782135, 0.44706071961879 , -0.145299823706503],
|
||||
[-0.25409760441282 , 0.46420381416882 , -0.208177722146526],
|
||||
[-0.322196658344302, 0.464439136821772, -0.250558059367669],
|
||||
[-0.385529968662985, 0.402800553948697, -0.310031082540741],
|
||||
[ 0. , 0.293332603215811, -0.137582088779393],
|
||||
[ 0. , 0.194828701837823, -0.069158109325951],
|
||||
[ 0. , 0.103844017393155, -0.009151819844964],
|
||||
[ 0. , 0. , 0. ],
|
||||
[ 0.080626352317973, -0.041276068128093, -0.134161035564826],
|
||||
[ 0.046439347377934, -0.057675223874769, -0.102990627164664],
|
||||
[ 0. , -0.068753126205604, -0.090545348482397],
|
||||
[-0.046439347377934, -0.057675223874769, -0.102990627164664],
|
||||
[-0.080626352317973, -0.041276068128093, -0.134161035564826],
|
||||
[ 0.315905195966084, 0.298337502555443, -0.285107407636464],
|
||||
[ 0.275252345439353, 0.312721904921771, -0.244558251170671],
|
||||
[ 0.176394511553111, 0.311907184376107, -0.219205360345231],
|
||||
[ 0.131229723798772, 0.284447361805627, -0.234239149487417],
|
||||
[ 0.184124948330084, 0.260179585304867, -0.226590776513707],
|
||||
[ 0.279433549294448, 0.267363071770222, -0.248441437111633],
|
||||
[-0.131229723798772, 0.284447361805627, -0.234239149487417],
|
||||
[-0.176394511553111, 0.311907184376107, -0.219205360345231],
|
||||
[-0.275252345439353, 0.312721904921771, -0.244558251170671],
|
||||
[-0.315905195966084, 0.298337502555443, -0.285107407636464],
|
||||
[-0.279433549294448, 0.267363071770222, -0.248441437111633],
|
||||
[-0.184124948330084, 0.260179585304867, -0.226590776513707],
|
||||
[ 0.121155252430729, -0.208988660580347, -0.160606287940521],
|
||||
[ 0.041356305910044, -0.194484199722098, -0.096159882202821],
|
||||
[ 0. , -0.205180167345702, -0.083299217789729],
|
||||
[-0.041356305910044, -0.194484199722098, -0.096159882202821],
|
||||
[-0.121155252430729, -0.208988660580347, -0.160606287940521],
|
||||
[-0.132325402795928, -0.290857984604968, -0.187067868218105],
|
||||
[-0.064137791831655, -0.325377847425684, -0.158924039726607],
|
||||
[ 0. , -0.343742581679188, -0.113925986025684],
|
||||
[ 0.064137791831655, -0.325377847425684, -0.158924039726607],
|
||||
[ 0.132325402795928, -0.290857984604968, -0.187067868218105],
|
||||
[ 0.181481567104525, -0.243239316141725, -0.231284988892766],
|
||||
[ 0.083999507750469, -0.239717753728704, -0.155256465640701],
|
||||
[ 0. , -0.256058040176369, -0.0950619498899 ],
|
||||
[-0.083999507750469, -0.239717753728704, -0.155256465640701],
|
||||
[-0.181481567104525, -0.243239316141725, -0.231284988892766],
|
||||
[-0.074036069749345, -0.250689938345682, -0.177346470406188],
|
||||
[ 0. , -0.264945854681568, -0.112349967428413],
|
||||
[ 0.074036069749345, -0.250689938345682, -0.177346470406188],
|
||||
# Pupils and eyeball centers
|
||||
[ 0.257990002632141, 0.276080012321472, -0.219998998939991],
|
||||
[-0.257990002632141, 0.276080012321472, -0.219998998939991],
|
||||
[ 0.257990002632141, 0.276080012321472, -0.324570998549461],
|
||||
[-0.257990002632141, 0.276080012321472, -0.324570998549461]
|
||||
], np.float32)
|
||||
def __init__(self, featureType):
|
||||
self.featureType = featureType
|
||||
self.face_3d = np.array([
|
||||
[ 0.4551769692672 , 0.300895790030204, -0.764429433974752],
|
||||
[ 0.448998827123556, 0.166995837790733, -0.765143004071253],
|
||||
[ 0.437431554952677, 0.022655479179981, -0.739267175112735],
|
||||
[ 0.415033422928434, -0.088941454648772, -0.747947437846473],
|
||||
[ 0.389123587370091, -0.232380029794684, -0.704788385327458],
|
||||
[ 0.334630113904382, -0.361265387599081, -0.615587579236862],
|
||||
[ 0.263725112132858, -0.460009725616771, -0.491479221041573],
|
||||
[ 0.16241621322721 , -0.558037146073869, -0.339445180872282],
|
||||
[ 0. , -0.621079019321682, -0.287294770748887],
|
||||
[-0.16241621322721 , -0.558037146073869, -0.339445180872282],
|
||||
[-0.263725112132858, -0.460009725616771, -0.491479221041573],
|
||||
[-0.334630113904382, -0.361265387599081, -0.615587579236862],
|
||||
[-0.389123587370091, -0.232380029794684, -0.704788385327458],
|
||||
[-0.415033422928434, -0.088941454648772, -0.747947437846473],
|
||||
[-0.437431554952677, 0.022655479179981, -0.739267175112735],
|
||||
[-0.448998827123556, 0.166995837790733, -0.765143004071253],
|
||||
[-0.4551769692672 , 0.300895790030204, -0.764429433974752],
|
||||
[ 0.385529968662985, 0.402800553948697, -0.310031082540741],
|
||||
[ 0.322196658344302, 0.464439136821772, -0.250558059367669],
|
||||
[ 0.25409760441282 , 0.46420381416882 , -0.208177722146526],
|
||||
[ 0.186875436782135, 0.44706071961879 , -0.145299823706503],
|
||||
[ 0.120880983543622, 0.423566314072968, -0.110757158774771],
|
||||
[-0.120880983543622, 0.423566314072968, -0.110757158774771],
|
||||
[-0.186875436782135, 0.44706071961879 , -0.145299823706503],
|
||||
[-0.25409760441282 , 0.46420381416882 , -0.208177722146526],
|
||||
[-0.322196658344302, 0.464439136821772, -0.250558059367669],
|
||||
[-0.385529968662985, 0.402800553948697, -0.310031082540741],
|
||||
[ 0. , 0.293332603215811, -0.137582088779393],
|
||||
[ 0. , 0.194828701837823, -0.069158109325951],
|
||||
[ 0. , 0.103844017393155, -0.009151819844964],
|
||||
[ 0. , 0. , 0. ],
|
||||
[ 0.080626352317973, -0.041276068128093, -0.134161035564826],
|
||||
[ 0.046439347377934, -0.057675223874769, -0.102990627164664],
|
||||
[ 0. , -0.068753126205604, -0.090545348482397],
|
||||
[-0.046439347377934, -0.057675223874769, -0.102990627164664],
|
||||
[-0.080626352317973, -0.041276068128093, -0.134161035564826],
|
||||
[ 0.315905195966084, 0.298337502555443, -0.285107407636464],
|
||||
[ 0.275252345439353, 0.312721904921771, -0.244558251170671],
|
||||
[ 0.176394511553111, 0.311907184376107, -0.219205360345231],
|
||||
[ 0.131229723798772, 0.284447361805627, -0.234239149487417],
|
||||
[ 0.184124948330084, 0.260179585304867, -0.226590776513707],
|
||||
[ 0.279433549294448, 0.267363071770222, -0.248441437111633],
|
||||
[-0.131229723798772, 0.284447361805627, -0.234239149487417],
|
||||
[-0.176394511553111, 0.311907184376107, -0.219205360345231],
|
||||
[-0.275252345439353, 0.312721904921771, -0.244558251170671],
|
||||
[-0.315905195966084, 0.298337502555443, -0.285107407636464],
|
||||
[-0.279433549294448, 0.267363071770222, -0.248441437111633],
|
||||
[-0.184124948330084, 0.260179585304867, -0.226590776513707],
|
||||
[ 0.121155252430729, -0.208988660580347, -0.160606287940521],
|
||||
[ 0.041356305910044, -0.194484199722098, -0.096159882202821],
|
||||
[ 0. , -0.205180167345702, -0.083299217789729],
|
||||
[-0.041356305910044, -0.194484199722098, -0.096159882202821],
|
||||
[-0.121155252430729, -0.208988660580347, -0.160606287940521],
|
||||
[-0.132325402795928, -0.290857984604968, -0.187067868218105],
|
||||
[-0.064137791831655, -0.325377847425684, -0.158924039726607],
|
||||
[ 0. , -0.343742581679188, -0.113925986025684],
|
||||
[ 0.064137791831655, -0.325377847425684, -0.158924039726607],
|
||||
[ 0.132325402795928, -0.290857984604968, -0.187067868218105],
|
||||
[ 0.181481567104525, -0.243239316141725, -0.231284988892766],
|
||||
[ 0.083999507750469, -0.239717753728704, -0.155256465640701],
|
||||
[ 0. , -0.256058040176369, -0.0950619498899 ],
|
||||
[-0.083999507750469, -0.239717753728704, -0.155256465640701],
|
||||
[-0.181481567104525, -0.243239316141725, -0.231284988892766],
|
||||
[-0.074036069749345, -0.250689938345682, -0.177346470406188],
|
||||
[ 0. , -0.264945854681568, -0.112349967428413],
|
||||
[ 0.074036069749345, -0.250689938345682, -0.177346470406188],
|
||||
# Pupils and eyeball centers
|
||||
[ 0.257990002632141, 0.276080012321472, -0.219998998939991],
|
||||
[-0.257990002632141, 0.276080012321472, -0.219998998939991],
|
||||
[ 0.257990002632141, 0.276080012321472, -0.324570998549461],
|
||||
[-0.257990002632141, 0.276080012321472, -0.324570998549461]
|
||||
], np.float32)
|
||||
|
||||
|
||||
|
||||
self.reset()
|
||||
|
|
|
|||
|
|
@ -1,34 +1,35 @@
|
|||
import numpy as np
|
||||
import cv2
|
||||
cv2.setNumThreads(6)
|
||||
|
||||
def detect_faces(frame, model, detection_threshold ):
|
||||
|
||||
image = resizeImage(frame)
|
||||
image = resizeImage(frame)
|
||||
|
||||
outputs, maxpool = model.faceDetection.run([], {'input': image})
|
||||
outputs = outputs[0]
|
||||
maxpool = maxpool[0]
|
||||
mask = outputs[0] == maxpool[0]
|
||||
mask2 = outputs[1] > 0.15
|
||||
outputs[0] = outputs[0] * mask.astype(int) * mask2.astype(int)
|
||||
faceLocation = np.argmax(outputs[0].flatten())
|
||||
x = faceLocation % 56
|
||||
y = faceLocation // 56
|
||||
if outputs[0, y, x] < detection_threshold:
|
||||
return None
|
||||
r = outputs[1, y, x] * 112.
|
||||
results= (((x * 4) - r, (y * 4) - r, r*2,r*2))
|
||||
results = np.array(results).astype(np.float32)
|
||||
results[[0,2]] *= frame.shape[1] / 224.
|
||||
results[[1,3]] *= frame.shape[0] / 224.
|
||||
outputs, maxpool = model.faceDetection.run([], {'input': image})
|
||||
outputs = outputs[0]
|
||||
maxpool = maxpool[0]
|
||||
mask = outputs[0] == maxpool[0]
|
||||
mask2 = outputs[1] > 0.15
|
||||
outputs[0] = outputs[0] * mask.astype(int) * mask2.astype(int)
|
||||
faceLocation = np.argmax(outputs[0].flatten())
|
||||
x = faceLocation % 56
|
||||
y = faceLocation // 56
|
||||
if outputs[0, y, x] < detection_threshold:
|
||||
return None
|
||||
r = outputs[1, y, x] * 112.
|
||||
results= (((x * 4) - r, (y * 4) - r, r*2,r*2))
|
||||
results = np.array(results).astype(np.float32)
|
||||
results[[0,2]] *= frame.shape[1] / 224.
|
||||
results[[1,3]] *= frame.shape[0] / 224.
|
||||
|
||||
return results
|
||||
return results
|
||||
|
||||
def resizeImage(frame):
|
||||
mean = np.float32(np.array([-2.1179, -2.0357, -1.8044]))
|
||||
std = np.float32(np.array([0.0171, 0.0175, 0.0174]))
|
||||
targetimageSize = [224, 224]
|
||||
image = cv2.resize(frame, targetimageSize, interpolation=cv2.INTER_LINEAR) * std + mean
|
||||
image = cv2.resize(frame, targetimageSize, interpolation=cv2.INTER_CUBIC) * std + mean
|
||||
image = np.expand_dims(image, 0)
|
||||
image = np.transpose(image, (0,3,1,2))
|
||||
return image
|
||||
|
|
|
|||
|
|
@ -26,14 +26,13 @@ parser.add_argument("--model", type=int, help="This can be used to select the tr
|
|||
parser.add_argument("--preview", type=int, help="Preview the frames sent to the tracker", default=0)
|
||||
parser.add_argument("--feature-type", type=int, help="Sets which version of feature extraction is used. 0 is my new version that works well for me and allows for some customization, 1 is EmilianaVT's version aka, normal OpenSeeFace operation", default=0, choices=[0, 1])
|
||||
parser.add_argument("--numpy-threads", type=int, help="Numer of threads Numpy can use, doesn't seem to effect much", default=1)
|
||||
parser.add_argument("-T","--threads", type=int, help="Numer of threads used for landmark detection. Default is 1 (~15ms per frame on my computer), 2 gets slightly faster frames (~10ms on my computer), more than 2 doesn't seem to help much", default=1)
|
||||
parser.add_argument("-T","--threads", type=int, help="Numer of threads used for landmark detection. Default is 1 (~15ms per frame on my computer), 2 gets slightly faster frames (~10ms on my computer), more than 2 doesn't seem to help much", default=2)
|
||||
parser.add_argument("-v", "--visualize", type=int, help="Set this to 1 to visualize the tracking", default=0)
|
||||
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
#processing args
|
||||
#For Vtube Studio: ip 127.0.0.1 port 11573
|
||||
target_ip = args.ip
|
||||
target_port = args.port
|
||||
fps = args.fps
|
||||
|
|
@ -53,7 +52,8 @@ silent = (args.silent == 1)
|
|||
if height > 480:
|
||||
frameQueueSize = 2
|
||||
else:
|
||||
frameQueueSize = 1
|
||||
frameQueueSize = 2
|
||||
|
||||
frameQueue = multiprocessing.Queue(maxsize=frameQueueSize)
|
||||
faceQueue = multiprocessing.Queue(maxsize=1)
|
||||
packetQueue = queue.Queue()
|
||||
|
|
@ -65,9 +65,7 @@ packetSenderThread.daemon = True
|
|||
packetSenderThread.start()
|
||||
|
||||
#creating processes
|
||||
#not a very active process, it's blocked by the main thread 90% of the time
|
||||
#but putting it in it's own process means there's a little bit less occupying the same core as the main thread
|
||||
#which helps when the system is under load
|
||||
|
||||
webcamProcess = multiprocessing.Process(target=webcam.startProcess,args = (frameQueue, faceQueue, fps, targetBrightness, width, height, mirror_input, ))
|
||||
webcamProcess.daemon = True
|
||||
webcamProcess.start()
|
||||
|
|
@ -119,23 +117,23 @@ try:
|
|||
if previewFrameQueue.qsize() < 1:
|
||||
previewFrameQueue.put(frame)
|
||||
|
||||
face, face_center, face_radius = tracker.predict(frame)
|
||||
faceInfo, face = tracker.predict(frame)
|
||||
|
||||
if face is not None:
|
||||
packet = VTS.preparePacket(face)
|
||||
if faceInfo is not None:
|
||||
packet = VTS.preparePacket(faceInfo)
|
||||
if faceQueue.qsize() < 1:
|
||||
faceQueue.put([face_center, face_radius])
|
||||
faceQueue.put(face)
|
||||
if visualizeFlag:
|
||||
preview.visualize(frame, face, previewFrameQueue, face_center, face_radius)
|
||||
preview.visualize(frame, faceInfo, previewFrameQueue, face_center, face_radius)
|
||||
|
||||
duration = time.perf_counter() - frame_get
|
||||
total_active_time += duration
|
||||
peak_frame_time = max(peak_frame_time, duration)
|
||||
|
||||
|
||||
sleepTimer = max(time.perf_counter() - frame_get, sleepTimer)
|
||||
time.sleep(max(peak_frame_time - duration, 0))
|
||||
sleepTimer -= 0.0005
|
||||
if duration < target_duration:
|
||||
time.sleep(target_duration - duration)
|
||||
else:
|
||||
#all caps because if this is consistently happening it's an issue
|
||||
messageQueue.put("CANNOT MAINTAIN FRAMERATE!")
|
||||
|
||||
peak_time_between = max(peak_time_between, time.perf_counter() -frame_start)
|
||||
total_run_time += time.perf_counter() -frame_start
|
||||
|
|
|
|||
0
image.py
Normal file
0
image.py
Normal file
|
|
@ -1,6 +1,7 @@
|
|||
import numpy as np
|
||||
import math
|
||||
import cv2
|
||||
cv2.setNumThreads(6)
|
||||
|
||||
def matrix_to_quaternion(m):
|
||||
t = 0.0
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import cv2
|
||||
import multiprocessing
|
||||
cv2.setNumThreads(6)
|
||||
|
||||
#this is it's own file just for ceanlieness
|
||||
|
||||
|
|
|
|||
54
tracker.py
54
tracker.py
|
|
@ -2,6 +2,7 @@ import os
|
|||
import numpy as np
|
||||
import onnxruntime
|
||||
import cv2
|
||||
cv2.setNumThreads(6)
|
||||
import time
|
||||
import facedetection
|
||||
import eyes
|
||||
|
|
@ -9,27 +10,34 @@ import landmarks
|
|||
import face
|
||||
|
||||
|
||||
|
||||
#this file is so much smaller than it used to be, lol
|
||||
#I moved most of the functionality into separate files so they're easier to work with
|
||||
|
||||
def clamp_to_im(pt, w, h): #8 times per frame, but that only accounts for 0.005ms
|
||||
x=max(pt[0],0)
|
||||
y=max(pt[1],0)
|
||||
def clamp_to_im(pt, w, h):
|
||||
x = pt[0]
|
||||
y = pt[1]
|
||||
if x < 0:
|
||||
x = 0
|
||||
if y < 0:
|
||||
y = 0
|
||||
if x >= w:
|
||||
x = w-1
|
||||
if y >= h:
|
||||
y = h-1
|
||||
return (int(x),int(y+1))
|
||||
return (int(x), int(y+1))
|
||||
|
||||
class Models():
|
||||
|
||||
models = [
|
||||
"lm_model0_opt.onnx",
|
||||
"lm_model1_opt.onnx",
|
||||
"lm_model2_opt.onnx",
|
||||
"lm_model3_opt.onnx",
|
||||
"lm_model4_opt.onnx"]
|
||||
|
||||
def __init__(self, threads, model_type=3):
|
||||
self.model_type = model_type
|
||||
self.models = [
|
||||
"lm_model0_opt.onnx",
|
||||
"lm_model1_opt.onnx",
|
||||
"lm_model2_opt.onnx",
|
||||
"lm_model3_opt.onnx",
|
||||
"lm_model4_opt.onnx"]
|
||||
|
||||
model = self.models[self.model_type]
|
||||
model_base_path = os.path.join(os.path.dirname(__file__), os.path.join("models"))
|
||||
|
|
@ -48,6 +56,9 @@ class Models():
|
|||
self.gazeTracker = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options, providers=providersList)
|
||||
|
||||
class Tracker():
|
||||
targetimageSize = [224, 224]
|
||||
std = np.float32(np.array([0.0171, 0.0175, 0.0174]))
|
||||
mean = np.float32(np.array([-2.1179, -2.0357, -1.8044]))
|
||||
def __init__(self, width, height, featureType, threads, model_type=3, detection_threshold=0.6, threshold=0.6, silent=False):
|
||||
|
||||
self.detection_threshold = detection_threshold
|
||||
|
|
@ -55,9 +66,6 @@ class Tracker():
|
|||
self.model = Models(threads, model_type = model_type)
|
||||
|
||||
# Image normalization constants
|
||||
self.mean = np.float32(np.array([-2.1179, -2.0357, -1.8044]))
|
||||
self.std = np.float32(np.array([0.0171, 0.0175, 0.0174]))
|
||||
self.targetimageSize = [224, 224]
|
||||
self.width = width
|
||||
self.height = height
|
||||
|
||||
|
|
@ -75,12 +83,10 @@ class Tracker():
|
|||
duration_pp = 0.0
|
||||
x,y,w,h = self.face
|
||||
|
||||
wint = int(w * 0.1)
|
||||
hint = int(h * 0.125)
|
||||
crop_x1 = x - wint
|
||||
crop_y1 = y - hint
|
||||
crop_x2 = x + w + wint
|
||||
crop_y2 = y + h + hint
|
||||
crop_x1 = x - (w * 0.1)
|
||||
crop_y1 = y - (h * 0.125)
|
||||
crop_x2 = x + w + (w * 0.1)
|
||||
crop_y2 = y + h + (h * 0.125)
|
||||
|
||||
crop_x1, crop_y1 = clamp_to_im((crop_x1, crop_y1), self.width, self.height)
|
||||
crop_x2, crop_y2 = clamp_to_im((crop_x2, crop_y2), self.width, self.height)
|
||||
|
|
@ -104,7 +110,7 @@ class Tracker():
|
|||
self.face = None
|
||||
duration = (time.perf_counter() - start) * 1000
|
||||
print(f"Took {duration:.2f}ms")
|
||||
return None, None, None
|
||||
return None, None
|
||||
|
||||
def predict(self, frame):
|
||||
start = time.perf_counter()
|
||||
|
|
@ -146,13 +152,13 @@ class Tracker():
|
|||
if face_info.success:
|
||||
face_info.adjust_3d()
|
||||
lms = face_info.lms[:, 0:2]
|
||||
x1, y1 = lms[0:66].min(0)
|
||||
x2, y2 = lms[0:66].max(0)
|
||||
self.face = [y1, x1, y2 - y1, x2 - x1]
|
||||
y1, x1 = lms[0:66].min(0)
|
||||
y2, x2 = lms[0:66].max(0)
|
||||
self.face = [x1, y1, x2 - x1, y2 - y1]
|
||||
duration_pnp += 1000 * (time.perf_counter() - start_pnp)
|
||||
duration = (time.perf_counter() - start) * 1000
|
||||
print(f"Took {duration:.2f}ms (detect: {duration_fd:.2f}ms, crop: {duration_pp:.2f}ms, track: {duration_model:.2f}ms, 3D points: {duration_pnp:.2f}ms)")
|
||||
return face_info, self.EyeTracker.faceCenter, self.EyeTracker.faceRadius
|
||||
return face_info, self.face
|
||||
|
||||
#Combined multiple failures into one catch all exit
|
||||
return self.early_exit("Face info not valid", start)
|
||||
|
|
|
|||
3
vts.py
3
vts.py
|
|
@ -3,13 +3,14 @@ import socket
|
|||
import struct
|
||||
|
||||
class VTS():
|
||||
|
||||
features = ["eye_l", "eye_r", "eyebrow_steepness_l", "eyebrow_updown_l", "eyebrow_quirk_l", "eyebrow_steepness_r", "eyebrow_updown_r", "eyebrow_quirk_r", "mouth_corner_updown_l", "mouth_corner_inout_l", "mouth_corner_updown_r", "mouth_corner_inout_r", "mouth_open", "mouth_wide"]
|
||||
def __init__(self, target_ip, target_port, silent, height, width, packetQueue):
|
||||
|
||||
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
self.target_ip = target_ip
|
||||
self.target_port = target_port
|
||||
self.silent = silent
|
||||
self.features = ["eye_l", "eye_r", "eyebrow_steepness_l", "eyebrow_updown_l", "eyebrow_quirk_l", "eyebrow_steepness_r", "eyebrow_updown_r", "eyebrow_quirk_r", "mouth_corner_updown_l", "mouth_corner_inout_l", "mouth_corner_updown_r", "mouth_corner_inout_r", "mouth_open", "mouth_wide"]
|
||||
self.width = width
|
||||
self.height = height
|
||||
self.packetQueue = packetQueue
|
||||
|
|
|
|||
35
webcam.py
35
webcam.py
|
|
@ -3,11 +3,24 @@ import os
|
|||
os.environ["OMP_NUM_THREADS"] = str(1)
|
||||
import dshowcapture
|
||||
import cv2
|
||||
cv2.setNumThreads(6)
|
||||
import multiprocessing
|
||||
import time
|
||||
import numpy as np
|
||||
import math
|
||||
|
||||
def clamp_to_im(pt, w, h):
|
||||
x = pt[0]
|
||||
y = pt[1]
|
||||
if x < 0:
|
||||
x = 0
|
||||
if y < 0:
|
||||
y = 0
|
||||
if x >= w:
|
||||
x = w-1
|
||||
if y >= h:
|
||||
y = h-1
|
||||
return (int(x), int(y+1))
|
||||
|
||||
#this is just here for conveience
|
||||
def startProcess(frameQueue, faceQueue, fps, targetBrightness, width, height, mirrorInput):
|
||||
|
|
@ -43,6 +56,7 @@ class Webcam():
|
|||
self.frame = None
|
||||
|
||||
def start(self):
|
||||
|
||||
while self.cap.isOpened():
|
||||
frameStart= time.perf_counter()
|
||||
totalFrameLatency = time.perf_counter()
|
||||
|
|
@ -91,17 +105,20 @@ class Webcam():
|
|||
#calculate the ideal gamma based on the brightness of the user's face
|
||||
#kind of winging it with the math, but it's a lot better than calculating based on the whole frame
|
||||
def updateGamma(self):
|
||||
face_center, face_radius = self.faceQueue.get()
|
||||
face = self.faceQueue.get()
|
||||
|
||||
y1 = int(face_center[1] - face_radius[1])
|
||||
y2 = int(face_center[1] + face_radius[1])
|
||||
x1 = int(face_center[0] - face_radius[0])
|
||||
x2 = int(face_center[0] + face_radius[0])
|
||||
x,y,w,h = face
|
||||
crop_x1 = x
|
||||
crop_y1 = y
|
||||
crop_x2 = x + w
|
||||
crop_y2 = y + h
|
||||
|
||||
#I was doing random sampling of the image brightness
|
||||
#and that's probably brighter for the whole frame
|
||||
#but for just the face this is pretty fast
|
||||
averageBrightness = np.mean(self.brightnessFrame[y1:y2,x1:x2])/256
|
||||
crop_x1, crop_y1 = clamp_to_im((crop_x1, crop_y1), self.width, self.height)
|
||||
crop_x2, crop_y2 = clamp_to_im((crop_x2, crop_y2), self.width, self.height)
|
||||
|
||||
croppedFace = self.brightnessFrame[crop_y1:crop_y2, crop_x1:crop_x2]
|
||||
|
||||
averageBrightness = np.mean(croppedFace)/256
|
||||
self.gamma = math.log(self.targetBrightness, averageBrightness)
|
||||
return
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue