copilot/OpenSeeFace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

This changes the points used for PnP solving and replaces the RetinaFace model with a higher resolution version.

Browse source
This commit is contained in:
Emiliana 2020-02-28 02:10:23 +01:00
parent 4a88c57be2
commit 033a48b132
8 changed files with 49 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

23
Examples/OpenSeeVRMDriver.cs
View file

@ -42,7 +42,8 @@ public class OpenSeeVRMDriver : MonoBehaviour {
new OpenSeeVRMExpression("fun", BlendShapePreset.Fun, 1f, 1f, true, true),
new OpenSeeVRMExpression("joy", BlendShapePreset.Joy, 1f, 1f, true, true),
new OpenSeeVRMExpression("angry", BlendShapePreset.Angry, 1f, 1f, true, true),
new OpenSeeVRMExpression("sorrow", BlendShapePreset.Sorrow, 1f, 1f, true, true)
new OpenSeeVRMExpression("sorrow", BlendShapePreset.Sorrow, 1f, 1f, true, true),
new OpenSeeVRMExpression("surprise", "Surprised", 1f, 1f, true, true)
};
[Tooltip("The expression configuration is initialized on startup. If it is changed and needs to be reinitialized, this can be triggered by using this flag or calling InitExpressionMap. This flag is reset to false afterwards.")]
public bool reloadExpressions = false;
@ -161,6 +162,8 @@ public class OpenSeeVRMDriver : MonoBehaviour {
public string trigger = "neutral";
[Tooltip("This is the VRM blend shape that will get triggered by the expression.")]
public BlendShapePreset blendShapePreset;
[Tooltip("When this name is set, it will be used to create a custom blendshape key.")]
public string customBlendShapeName = "";
[Tooltip("The weight determines the value the expression blend shape should be set to.")]
[Range(0, 1)]
public float weight = 1f;
@ -182,11 +185,23 @@ public class OpenSeeVRMDriver : MonoBehaviour {
this.enableVisemes = visemes;
this.enableBlinking = blinking;
}
public OpenSeeVRMExpression(string trigger, string name, float weight, float factor, bool visemes, bool blinking) {
this.trigger = trigger;
this.customBlendShapeName = name;
this.weight = weight;
this.visemeFactor = factor;
this.enableVisemes = visemes;
this.enableBlinking = blinking;
}
}
public void InitExpressionMap() {
expressionMap = new Dictionary<string, OpenSeeVRMExpression>();
foreach (var expression in expressions) {
if (expression.customBlendShapeName != "")
expression.blendShapeKey = new BlendShapeKey(expression.customBlendShapeName);
else
expression.blendShapeKey = new BlendShapeKey(expression.blendShapePreset);
expressionMap.Add(expression.trigger, expression);
}
@ -216,7 +231,7 @@ public class OpenSeeVRMDriver : MonoBehaviour {
vrmBlendShapeProxy.ImmediatelySetValue(currentExpression.blendShapeKey, currentExpression.weight);
}
void GetLookParameters(ref float lookLeftRight, ref float lookUpDown, bool update, int gazePoint, int topRight, int topLeft, int bottomRight, int bottomLeft) {
void GetLookParameters(ref float lookLeftRight, ref float lookUpDown, bool update, int gazePoint, int right, int left, int topRight, int topLeft, int bottomRight, int bottomLeft) {
float borderRight = (openSeeData.points3D[topRight].x + openSeeData.points3D[bottomRight].x) / 2.0f;
float borderLeft = (openSeeData.points3D[topLeft].x + openSeeData.points3D[bottomLeft].x) / 2.0f;
float horizontalCenter = (borderRight + borderLeft) / 2.0f;
@ -248,8 +263,8 @@ public class OpenSeeVRMDriver : MonoBehaviour {
float lookLeftRight = currentLookLeftRight;
if (lastGaze < openSeeData.time) {
GetLookParameters(ref lookLeftRight, ref lookUpDown, false, 66, 37, 38, 41, 40);
GetLookParameters(ref lookLeftRight, ref lookUpDown, true, 67, 43, 44, 47, 46);
GetLookParameters(ref lookLeftRight, ref lookUpDown, false, 66, 36, 39, 37, 38, 41, 40);
GetLookParameters(ref lookLeftRight, ref lookUpDown, true, 67, 42, 45, 43, 44, 47, 46);
lookLeftRight = Mathf.Lerp(currentLookLeftRight, lookLeftRight, 1f - gazeSmoothing);
lookUpDown = Mathf.Lerp(currentLookUpDown, lookUpDown, 1f - gazeSmoothing);

2
README.md
View file

@ -150,7 +150,7 @@ Face detection is done using OpenCV's Haar Cascade classifier or RetinaFace.
year={2019}
}
RetinaFace detection is based on [this](https://github.com/biubug6/Pytorch_Retinaface) implementation. The pretrained model was modified to remove unnecessary landmark detection and converted to ONNX format for a resolution of 384x384.
RetinaFace detection is based on [this](https://github.com/biubug6/Pytorch_Retinaface) implementation. The pretrained model was modified to remove unnecessary landmark detection and converted to ONNX format for a resolution of 640x640.
# License

3
facetracker.py
View file

@ -29,7 +29,6 @@ parser.add_argument("--raw-rgb", type=int, help="When this is set, raw RGB frame
parser.add_argument("--log-data", help="You can set a filename to which tracking data will be logged here", default="")
parser.add_argument("--model", type=int, help="This can be used to select the tracking model. Higher numbers are models with better tracking quality, but slower speed. The recommended models are 3 and 1.", default=3, choices=[0, 1, 2, 3])
parser.add_argument("--model-dir", help="This can be used to specify the path to the directory containing the .onnx model files", default=None)
parser.add_argument("--high-quality-3d", type=int, help="When set to 1, more nose points are used when estimating the face pose", default=1)
parser.add_argument("--gaze-tracking", type=int, help="When set to 1, experimental blink detection and gaze tracking are enabled, which makes things slightly slower", default=1)
parser.add_argument("--face-id-offset", type=int, help="When set, this offset is added to all face ids, which can be useful for mixing tracking data from multiple network sources", default=0)
args = parser.parse_args()
@ -107,7 +106,7 @@ try:
first = False
height, width, channels = frame.shape
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
tracker = Tracker(width, height, threshold=args.threshold, max_threads=args.max_threads, max_faces=args.faces, discard_after=args.discard_after, scan_every=args.scan_every, silent=False if args.silent == 0 else True, model_type=args.model, pnp_quality=args.high_quality_3d, model_dir=args.model_dir, no_gaze=False if args.gaze_tracking != 0 else True, use_retinaface=args.scan_retinaface)
tracker = Tracker(width, height, threshold=args.threshold, max_threads=args.max_threads, max_faces=args.faces, discard_after=args.discard_after, scan_every=args.scan_every, silent=False if args.silent == 0 else True, model_type=args.model, model_dir=args.model_dir, no_gaze=False if args.gaze_tracking != 0 else True, use_retinaface=args.scan_retinaface)
if not args.video_out is None:
out = cv2.VideoWriter(args.video_out, cv2.VideoWriter_fourcc('F','F','V','1'), 24, (width,height))

1
models/priorbox_384.json
View file

File diff suppressed because one or more lines are too long

1
models/priorbox_640x640.json Normal file
View file

File diff suppressed because one or more lines are too long

BIN
models/retinaface_opt.onnx → models/retinaface_640x640_opt.onnx
View file

Binary file not shown.

22
retinaface.py
View file

@ -59,7 +59,7 @@ def worker_thread(rfd, frame):
rfd.running = False
class RetinaFaceDetector():
def __init__(self, model_path="models/retinaface_opt.onnx", json_path="models/priorbox_384.json", threads=4, min_conf=0.3, nms_threshold=0.4, top_k=1):
def __init__(self, model_path="models/retinaface_640x640_opt.onnx", json_path="models/priorbox_640x640.json", threads=4, min_conf=0.4, nms_threshold=0.4, top_k=1, res=(640, 640)):
options = onnxruntime.SessionOptions()
options.inter_op_num_threads = 1
options.intra_op_num_threads = threads
@ -67,7 +67,7 @@ class RetinaFaceDetector():
options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
options.log_severity_level = 3
self.session = onnxruntime.InferenceSession(model_path, sess_options=options)
self.res = 384
self.res_w, self.res_h = res
with open(json_path, "r") as prior_file:
self.priorbox = np.array(json.loads(prior_file.read()))
self.min_conf = min_conf
@ -81,18 +81,18 @@ class RetinaFaceDetector():
h, w, _ = frame.shape
im = None
if h > w:
resize = h / self.res
im = cv2.resize(frame, (int(w / resize), self.res), interpolation=cv2.INTER_LINEAR)
resize = h / self.res_h
im = cv2.resize(frame, (int(w / resize), self.res_h), interpolation=cv2.INTER_LINEAR)
resize = 1 / resize
else:
resize = w / self.res
im = cv2.resize(frame, (self.res, int(h / resize)), interpolation=cv2.INTER_LINEAR)
resize = w / self.res_w
im = cv2.resize(frame, (self.res_w, int(h / resize)), interpolation=cv2.INTER_LINEAR)
resize = 1 / resize
pad_h = self.res - im.shape[0]
pad_w = self.res - im.shape[1]
pad_h = self.res_h - im.shape[0]
pad_w = self.res_w - im.shape[1]
im = cv2.copyMakeBorder(im, 0, pad_h, 0, pad_w, cv2.BORDER_CONSTANT, (0, 0, 0))
im = np.float32(im)
scale = np.array((self.res, self.res, self.res, self.res))
scale = np.array((self.res_w, self.res_h, self.res_w, self.res_h))
im -= (104, 117, 123)
im = im.transpose(2, 0, 1)
im = np.expand_dims(im, 0)
@ -112,8 +112,8 @@ class RetinaFaceDetector():
dets = dets[:self.top_k, 0:4]
dets[:, 2:4] = dets[:, 2:4] - dets[:, 0:2]
if is_background:
upsize = dets[:, 2:4] * np.array([[0.1, 0.0]])
if True:#is_background:
upsize = dets[:, 2:4] * np.array([[0.15, 0.2]])
dets[:, 0:2] -= upsize
dets[:, 2:4] += upsize * 2

40
tracker.py
View file

@ -120,14 +120,15 @@ class FaceInfo():
self.id = id
self.frame_count = -1
self.tracker = tracker
self.contour_pts = [0,1,15,16,27,28,29,30,31,32,33,34,35,36,39,42,45]
self.face_3d = copy.copy(self.tracker.face_3d)
self.reset()
self.alive = False
self.coord = None
self.limit_3d_adjustment = False
self.update_count_delta = 50.
self.update_count_max = 1500.
self.limit_3d_adjustment = True
self.update_count_delta = 75.
self.update_count_max = 2500.
def reset(self):
self.alive = False
@ -157,11 +158,7 @@ class FaceInfo():
self.alive = True
def update_contour(self):
self.contour[0:17] = self.face_3d[0:17]
if self.tracker.high_quality_3d:
self.contour[17:21] = self.face_3d[27:31]
else:
self.contour[17] = self.face_3d[30]
self.contour = np.array(self.face_3d[self.contour_pts])
def adjust_3d(self):
r = 1.0 + np.random.random_sample((66,3)) * 0.02 - 0.01
@ -182,16 +179,14 @@ class FaceInfo():
r[17:22, 2] = 1.0
r[31:33, 2] = 1.0
r[36:42, 2] = 1.0
for i in [48, 49, 56, 57, 58, 59, 65]:
r[i, 2] = 1.0
r[[48, 49, 56, 57, 58, 59, 65], 2] = 1.0
else:
update_type = 1
r[8:17, 2] = 1.0
r[22:27, 2] = 1.0
r[34:36, 2] = 1.0
r[42:48, 2] = 1.0
for i in [51, 52, 53, 54, 61, 62, 63]:
r[i, 2] = 1.0
r[[51, 52, 53, 54, 61, 62, 63], 2] = 1.0
c = self.face_3d[0:66] * r
o_projected = np.squeeze(np.array(cv2.projectPoints(self.face_3d[0:66], self.rotation, self.translation, self.tracker.camera, self.tracker.dist_coeffs)[0]), 1)
@ -227,7 +222,7 @@ class FaceInfo():
class Tracker():
def __init__(self, width, height, model_type=3, threshold=0.4, max_faces=1, discard_after=5, scan_every=3, bbox_growth=0.0, max_threads=4, silent=False, pnp_quality=1, model_dir=None, no_gaze=False, use_retinaface=False):
def __init__(self, width, height, model_type=3, threshold=0.4, max_faces=1, discard_after=5, scan_every=3, bbox_growth=0.0, max_threads=4, silent=False, model_dir=None, no_gaze=False, use_retinaface=False):
options = onnxruntime.SessionOptions()
options.inter_op_num_threads = 1
options.intra_op_num_threads = max_threads
@ -249,8 +244,8 @@ class Tracker():
else:
model_base_path = model_dir
self.retinaface = RetinaFaceDetector(model_path=os.path.join(model_base_path, "retinaface_opt.onnx"), json_path=os.path.join(model_base_path, "priorbox_384.json"), threads=max_threads, top_k=max_faces)
self.retinaface_slow = RetinaFaceDetector(model_path=os.path.join(model_base_path, "retinaface_opt.onnx"), json_path=os.path.join(model_base_path, "priorbox_384.json"), threads=2, top_k=max_faces)
self.retinaface = RetinaFaceDetector(model_path=os.path.join(model_base_path, "retinaface_640x640_opt.onnx"), json_path=os.path.join(model_base_path, "priorbox_640x640.json"), threads=max_threads, top_k=max_faces, res=(640, 640))
self.retinaface_scan = RetinaFaceDetector(model_path=os.path.join(model_base_path, "retinaface_640x640_opt.onnx"), json_path=os.path.join(model_base_path, "priorbox_640x640.json"), threads=2, top_k=max_faces, res=(640, 640))
self.use_retinaface = use_retinaface
# Single face instance with multiple threads
@ -292,7 +287,6 @@ class Tracker():
self.std = self.std * 255.0
# PnP solving
self.high_quality_3d = False if pnp_quality == 0 else True
self.face_3d = np.array([
[0.50194, 0.28030, -0.49561],
[0.48377, 0.15031, -0.50000],
@ -413,12 +407,7 @@ class Tracker():
def estimate_depth(self, face_info):
lms = np.concatenate((face_info.lms, np.array([[face_info.eye_state[0][1], face_info.eye_state[0][2], face_info.eye_state[0][3]], [face_info.eye_state[1][1], face_info.eye_state[1][2], face_info.eye_state[1][3]]], np.float)), 0)
image_pts = np.empty((21 if self.high_quality_3d else 18, 2), np.float32)
image_pts[0:17] = np.array(lms)[0:17,0:2]
if self.high_quality_3d:
image_pts[17:21] = np.array(lms)[27:31,0:2]
else:
image_pts[17] = np.array(lms)[30,0:2]
image_pts = np.array(lms)[face_info.contour_pts, 0:2]
success = False
if not face_info.rotation is None:
@ -671,15 +660,16 @@ class Tracker():
self.wait_count += 1
if self.detected == 0:
start_fd = time.perf_counter()
new_faces.extend(self.retinaface.detect_retina(frame))
retinaface_detections = self.retinaface.detect_retina(frame)
new_faces.extend(retinaface_detections)
duration_fd = 1000 * (time.perf_counter() - start_fd)
self.wait_count = 0
elif self.detected < self.max_faces:
if self.use_retinaface > 0:
new_faces.extend(self.retinaface_slow.get_results())
new_faces.extend(self.retinaface_scan.get_results())
if self.wait_count >= self.scan_every:
if self.use_retinaface > 0:
self.retinaface_slow.background_detect(frame)
self.retinaface_scan.background_detect(frame)
else:
start_fd = time.perf_counter()
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)