All models were retrained to fix bugs in the training process. There are also other bug fixes and improvements.
This commit is contained in:
Emiliana
parent
2595fa31a9
commit
dda7dd0519
6 changed files with 47 additions and 60 deletions
|
|
@ -101,13 +101,13 @@ public class OpenSee : MonoBehaviour {
|
|||
rawEuler = readVector3(b, ref o);
|
||||
|
||||
rotation = rawEuler;
|
||||
rotation.z = (rotation.z + 90) % 360;
|
||||
rotation.z = (rotation.z - 90) % 360;
|
||||
rotation.x = -(rotation.x + 180) % 360;
|
||||
|
||||
float x = readFloat(b, ref o);
|
||||
float y = readFloat(b, ref o);
|
||||
float z = readFloat(b, ref o);
|
||||
translation = new Vector3(y, -x, z);
|
||||
translation = new Vector3(-y, x, -z);
|
||||
|
||||
for (int i = 0; i < nPoints; i++) {
|
||||
confidence[i] = readFloat(b, ref o);
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ if os.name == 'nt':
|
|||
else:
|
||||
parser.add_argument("-W", "--width", type=int, help="Set raw RGB width", default=640)
|
||||
parser.add_argument("-H", "--height", type=int, help="Set raw RGB height", default=360)
|
||||
parser.add_argument("-c", "--capture", help="Set camera ID (0, 1...) or video file", default=0)
|
||||
parser.add_argument("-c", "--capture", help="Set camera ID (0, 1...) or video file", default="0")
|
||||
parser.add_argument("-m", "--max-threads", type=int, help="Set the maximum number of threads", default=1)
|
||||
parser.add_argument("-t", "--threshold", type=float, help="Set minimum confidence threshold for face detection", default=0.4)
|
||||
parser.add_argument("-v", "--visualize", type=int, help="Set this to 1 to visualize the tracking or to 2 to add numbers to the point display", default=0)
|
||||
|
|
@ -79,7 +79,7 @@ frame_count = 0
|
|||
|
||||
if args.log_data != "":
|
||||
log = open(args.log_data, "w")
|
||||
log.write("Frame,Time,Face,AverageConfidence,Success3D,PnPError,RotationQuat.X,RotationQuat.Y,RotationQuat.Z,RotationQuat.W,RVec.X,RVec.Y,RVec.Z,TVec.X,TVec.Y,TVec.Z")
|
||||
log.write("Frame,Time,Face,RightOpen,LeftOpen,AverageConfidence,Success3D,PnPError,RotationQuat.X,RotationQuat.Y,RotationQuat.Z,RotationQuat.W,RVec.X,RVec.Y,RVec.Z,TVec.X,TVec.Y,TVec.Z")
|
||||
for i in range(66):
|
||||
log.write(f",Landmark[{i}].X,Landmark[{i}].Y,Landmark[{i}].Confidence")
|
||||
for i in range(66):
|
||||
|
|
@ -108,16 +108,18 @@ try:
|
|||
if not args.video_out is None:
|
||||
out = cv2.VideoWriter(args.video_out, cv2.VideoWriter_fourcc('M','J','P','G'), 24, (width,height))
|
||||
|
||||
inference_start = time.time()
|
||||
inference_start = time.perf_counter()
|
||||
faces = tracker.predict(frame)
|
||||
if len(faces) > 0:
|
||||
tracking_time += (time.time() - inference_start) / len(faces)
|
||||
tracking_time += (time.perf_counter() - inference_start) / len(faces)
|
||||
tracking_frames += 1
|
||||
packet = bytearray()
|
||||
detected = False
|
||||
for face_num, (conf, lms, success_3d, pnp_error, quaternion, euler, rotation, translation, pts_3d, (right_open, left_open)) in enumerate(faces): # TODO: eye_state
|
||||
for face_num, (conf, lms, success_3d, pnp_error, quaternion, euler, rotation, translation, pts_3d, (right_open, left_open), (face_y, face_x, face_height, face_width)) in enumerate(faces):
|
||||
left_state = "O" if left_open > 0.5 else "-"
|
||||
right_state = "O" if right_open > 0.5 else "-"
|
||||
if args.silent == 0:
|
||||
print(f"Confidence: {conf:.4f} / 3D fitting error: {pnp_error:.4f}")
|
||||
print(f"Confidence: {conf:.4f} / 3D fitting error: {pnp_error:.4f} / Eyes: {left_state}, {right_state}")
|
||||
detected = True
|
||||
if not success_3d:
|
||||
pts_3d = np.zeros((70, 3), np.float32)
|
||||
|
|
@ -133,11 +135,11 @@ try:
|
|||
packet.extend(bytearray(struct.pack("f", euler[0])))
|
||||
packet.extend(bytearray(struct.pack("f", euler[1])))
|
||||
packet.extend(bytearray(struct.pack("f", euler[2])))
|
||||
packet.extend(bytearray(struct.pack("f", translation[0][0])))
|
||||
packet.extend(bytearray(struct.pack("f", translation[0][1])))
|
||||
packet.extend(bytearray(struct.pack("f", translation[0][2])))
|
||||
packet.extend(bytearray(struct.pack("f", translation[0])))
|
||||
packet.extend(bytearray(struct.pack("f", translation[1])))
|
||||
packet.extend(bytearray(struct.pack("f", translation[2])))
|
||||
if not log is None:
|
||||
log.write(f"{frame_count},{now},{face_num},{conf},{success_3d},{pnp_error},{quaternion[0]},{quaternion[1]},{quaternion[2]},{quaternion[3]},{rotation[0][0]},{rotation[0][1]},{rotation[0][2]},{translation[0][0]},{translation[0][1]},{translation[0][2]}")
|
||||
log.write(f"{frame_count},{now},{face_num},{right_open},{left_open},{conf},{success_3d},{pnp_error},{quaternion[0]},{quaternion[1]},{quaternion[2]},{quaternion[3]},{rotation[0][0]},{rotation[0][1]},{rotation[0][2]},{translation[0][0]},{translation[0][1]},{translation[0][2]}")
|
||||
for (x,y,c) in lms:
|
||||
packet.extend(bytearray(struct.pack("f", c)))
|
||||
for pt_num, (x,y,c) in enumerate(lms):
|
||||
|
|
@ -194,7 +196,7 @@ try:
|
|||
out.write(frame)
|
||||
|
||||
if args.visualize != 0:
|
||||
cv2.imshow('Frame', frame)
|
||||
cv2.imshow('OpenSeeFace Visualization', frame)
|
||||
if cv2.waitKey(1) & 0xFF == ord('q'):
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
Binary file not shown.
79
tracker.py
79
tracker.py
|
|
@ -1,5 +1,6 @@
|
|||
import os
|
||||
import numpy as np
|
||||
import math
|
||||
import cv2
|
||||
import onnxruntime
|
||||
import time
|
||||
|
|
@ -218,7 +219,7 @@ class Tracker():
|
|||
[-0.25799, 0.27608, -0.16923],
|
||||
[0.25799, 0.27608, -0.24967],
|
||||
[-0.25799, 0.27608, -0.24967],
|
||||
], np.float32) * np.array([1.0, 1.0, 1.15])
|
||||
], np.float32) * np.array([1.0, 1.0, 1.3])
|
||||
|
||||
self.contour = np.empty((21 if self.high_quality_3d else 18, 3))
|
||||
self.contour[0:17] = self.face_3d[0:17]
|
||||
|
|
@ -256,23 +257,10 @@ class Tracker():
|
|||
y = m % 28
|
||||
conf = float(tensor[i][x,y])
|
||||
avg_conf = avg_conf + conf
|
||||
|
||||
# Bugfix for heatmaps being offset during training (models need retraining, but this seems to be working fine somehow)
|
||||
x += 2
|
||||
y += 2
|
||||
if x >= 28:
|
||||
x = 27
|
||||
if y >= 28:
|
||||
y = 27
|
||||
|
||||
off_x = 0
|
||||
off_y = 0
|
||||
if self.model_type > 0:
|
||||
off_x = self.res * ((1. * logit(tensor[66 + i][y, x])) - 0.0)
|
||||
off_y = self.res * ((1. * logit(tensor[66 * 2 + i][y, x])) - 0.0)
|
||||
else:
|
||||
off_x = self.res * ((2. * float(tensor[66 + i][y, x])) - 1.0)
|
||||
off_y = self.res * ((2. * float(tensor[66 * 2 + i][y, x])) - 1.0)
|
||||
off_x = self.res * ((1. * logit(tensor[66 + i][x, y])) - 0.0)
|
||||
off_y = self.res * ((1. * logit(tensor[66 * 2 + i][x, y])) - 0.0)
|
||||
off_x = math.floor(off_x + 0.5)
|
||||
off_y = math.floor(off_y + 0.5)
|
||||
lm_x = crop_y1 + scale_y * (self.res * (float(x) / 28.) + off_x)
|
||||
lm_y = crop_x1 + scale_x * (self.res * (float(y) / 28.) + off_y)
|
||||
lms.append((lm_x,lm_y,conf))
|
||||
|
|
@ -291,19 +279,19 @@ class Tracker():
|
|||
|
||||
success = False
|
||||
if self.max_faces < 2 and not self.rotation is None:
|
||||
success, self.rotation, self.translation, inliers = cv2.solvePnPRansac(self.contour, image_pts, self.camera, self.dist_coeffs, useExtrinsicGuess=True, rvec=np.transpose(self.rotation), tvec=np.transpose(self.translation), flags=cv2.SOLVEPNP_ITERATIVE)
|
||||
success, self.rotation, self.translation = cv2.solvePnP(self.contour, image_pts, self.camera, self.dist_coeffs, useExtrinsicGuess=True, rvec=np.transpose(self.rotation), tvec=np.transpose(self.translation), flags=cv2.SOLVEPNP_ITERATIVE)
|
||||
else:
|
||||
rvec = np.array([-180, 0, -90], np.float32)
|
||||
rvec = np.array([0, 0, 0], np.float32)
|
||||
tvec = np.array([0, 0, 0], np.float32)
|
||||
success, self.rotation, self.translation, inliers = cv2.solvePnPRansac(self.contour, image_pts, self.camera, self.dist_coeffs, useExtrinsicGuess=True, rvec=rvec, tvec=tvec, flags=cv2.SOLVEPNP_ITERATIVE)
|
||||
success, self.rotation, self.translation = cv2.solvePnP(self.contour, image_pts, self.camera, self.dist_coeffs, useExtrinsicGuess=True, rvec=rvec, tvec=tvec, flags=cv2.SOLVEPNP_ITERATIVE)
|
||||
|
||||
rotation = self.rotation
|
||||
translation = self.translation
|
||||
|
||||
pts_3d = np.zeros((70,3), np.float32)
|
||||
if not success:
|
||||
self.rotation = np.array([[0.0, 0.0, 0.0]], np.float32)
|
||||
self.translation = np.array([[0.0, 0.0, 0.0]], np.float32)
|
||||
self.rotation = np.array([0.0, 0.0, 0.0], np.float32)
|
||||
self.translation = np.array([0.0, 0.0, 0.0], np.float32)
|
||||
return False, np.zeros(4), np.zeros(3), 99999., pts_3d, lms
|
||||
else:
|
||||
self.rotation = np.transpose(self.rotation)
|
||||
|
|
@ -325,7 +313,7 @@ class Tracker():
|
|||
continue
|
||||
reference = rmat.dot(pt)
|
||||
reference = reference + self.translation
|
||||
reference = self.camera.dot(reference[0])
|
||||
reference = self.camera.dot(reference)
|
||||
depth = reference[2]
|
||||
if i < 17 or i == 30:
|
||||
reference = reference / depth
|
||||
|
|
@ -335,7 +323,7 @@ class Tracker():
|
|||
pt_3d = np.array([lms[i][0] * depth, lms[i][1] * depth, depth], np.float32)
|
||||
pt_3d = self.inverse_camera.dot(pt_3d)
|
||||
pt_3d = pt_3d - self.translation
|
||||
pt_3d = inverse_rotation.dot(pt_3d[0])
|
||||
pt_3d = inverse_rotation.dot(pt_3d)
|
||||
pts_3d[i,:] = pt_3d[:]
|
||||
|
||||
euler = cv2.RQDecomp3x3(rmat)[0]
|
||||
|
|
@ -402,7 +390,7 @@ class Tracker():
|
|||
return eye_state
|
||||
|
||||
def predict(self, frame, additional_faces=[]):
|
||||
start = time.time()
|
||||
start = time.perf_counter()
|
||||
im = frame
|
||||
|
||||
duration_fd = 0.0
|
||||
|
|
@ -416,16 +404,16 @@ class Tracker():
|
|||
self.wait_count += 1
|
||||
if self.detected < self.max_faces:
|
||||
if self.detected == 0 or self.wait_count >= self.scan_every:
|
||||
start_fd = time.time()
|
||||
start_fd = time.perf_counter()
|
||||
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
||||
new_faces.extend(list(self.faceCascade.detectMultiScale(gray, 1.3, 4, 0 | cv2.CASCADE_SCALE_IMAGE, (50, 50))))
|
||||
duration_fd = 1000 * (time.time() - start_fd)
|
||||
duration_fd = 1000 * (time.perf_counter() - start_fd)
|
||||
self.wait_count = 0
|
||||
else:
|
||||
self.wait_count = 0
|
||||
|
||||
if len(new_faces) < 1:
|
||||
duration = (time.time() - start) * 1000
|
||||
duration = (time.perf_counter() - start) * 1000
|
||||
if not self.silent:
|
||||
print(f"Took {duration:.2f}ms")
|
||||
return []
|
||||
|
|
@ -434,16 +422,11 @@ class Tracker():
|
|||
crop_info = []
|
||||
num_crops = 0
|
||||
for j, (x,y,w,h) in enumerate(new_faces):
|
||||
if self.model_type < 2:
|
||||
crop_x1 = x - int(w * 0.2)
|
||||
crop_y1 = y - int(h * 0.25)
|
||||
crop_x2 = x + w + int(w * 0.2)
|
||||
crop_y2 = y + h + int(h * 0.25)
|
||||
else:
|
||||
crop_x1 = x - int(w * 0.1)
|
||||
crop_y1 = y - int(h * 0.1)
|
||||
crop_x2 = x + w + int(w * 0.1)
|
||||
crop_y2 = y + h + int(h * 0.1)
|
||||
crop_x1 = x - int(w * 0.1)
|
||||
crop_y1 = y - int(h * 0.125)
|
||||
crop_x2 = x + w + int(w * 0.1)
|
||||
crop_y2 = y + h + int(h * 0.125)
|
||||
|
||||
crop_x1, crop_y1 = clamp_to_im((crop_x1, crop_y1), self.width, self.height)
|
||||
crop_x2, crop_y2 = clamp_to_im((crop_x2, crop_y2), self.width, self.height)
|
||||
|
||||
|
|
@ -453,16 +436,16 @@ class Tracker():
|
|||
if crop_x2 - crop_x1 < 4 or crop_y2 - crop_y1 < 4:
|
||||
continue
|
||||
|
||||
start_pp = time.time()
|
||||
start_pp = time.perf_counter()
|
||||
crop = self.preprocess(im, (crop_x1, crop_y1, crop_x2, crop_y2))
|
||||
duration_pp += 1000 * (time.time() - start_pp)
|
||||
duration_pp += 1000 * (time.perf_counter() - start_pp)
|
||||
crops.append(crop)
|
||||
crop_info.append((crop_x1, crop_y1, scale_x, scale_y))
|
||||
num_crops += 1
|
||||
|
||||
groups = group_rects(new_faces)
|
||||
|
||||
start_model = time.time()
|
||||
start_model = time.perf_counter()
|
||||
outputs = {}
|
||||
if num_crops == 1:
|
||||
output = self.session.run([], {self.input_name: crops[0]})[0]
|
||||
|
|
@ -499,7 +482,7 @@ class Tracker():
|
|||
best_results[group_id][1] = (lms, eye_state)
|
||||
|
||||
sorted_results = sorted(best_results.values(), key=lambda x: x[0], reverse=True)
|
||||
duration_model = 1000 * (time.time() - start_model)
|
||||
duration_model = 1000 * (time.perf_counter() - start_model)
|
||||
|
||||
results = []
|
||||
detected = []
|
||||
|
|
@ -509,10 +492,9 @@ class Tracker():
|
|||
break;
|
||||
if conf > self.threshold:
|
||||
detections += 1
|
||||
start_pnp = time.time()
|
||||
start_pnp = time.perf_counter()
|
||||
success, quaternion, euler, pnp_error, pts_3d, lms = self.estimateDepth(lms, eye_state)
|
||||
duration_pnp += 1000 * (time.time() - start_pnp)
|
||||
results.append((conf, lms, success, pnp_error, quaternion, euler, self.rotation, self.translation, pts_3d, (eye_state[0][0], eye_state[1][0])))
|
||||
duration_pnp += 1000 * (time.perf_counter() - start_pnp)
|
||||
min_x = 10000
|
||||
max_x = -1
|
||||
min_y = 10000
|
||||
|
|
@ -529,6 +511,7 @@ class Tracker():
|
|||
w = max_x - min_x
|
||||
y = max_y - min_y
|
||||
detected.append((min_y, min_x, max_y - min_y, max_x - min_x))
|
||||
results.append((conf, lms, success, pnp_error, quaternion, euler, self.rotation, self.translation, pts_3d, (eye_state[0][0], eye_state[1][0]), (min_y, min_x, max_y - min_y, max_x - min_x)))
|
||||
|
||||
if len(detected) > 0:
|
||||
self.detected = len(detected)
|
||||
|
|
@ -552,8 +535,10 @@ class Tracker():
|
|||
faces.append((x,y,w,h))
|
||||
self.faces = faces
|
||||
|
||||
duration = (time.time() - start) * 1000
|
||||
duration = (time.perf_counter() - start) * 1000
|
||||
if not self.silent:
|
||||
print(f"Took {duration:.2f}ms (detect: {duration_fd:.2f}ms, crop: {duration_pp:.2f}, track: {duration_model:.2f}ms, 3D points: {duration_pnp:.2f}ms)")
|
||||
|
||||
results = sorted(results, key=lambda x: x[10][1] * self.height + x[10][0])
|
||||
|
||||
return results
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue