copilot/OpenSeeFace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Face 3D point adaption can be disabled and feature adaptability can be limited now.

Browse source
This commit is contained in:
Emiliana 2020-06-24 12:29:33 +02:00
parent 161363682d
commit 0690bdd15e
2 changed files with 114 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

4
facetracker.py
View file

@ -24,6 +24,8 @@ parser.add_argument("--faces", type=int, help="Set the maximum number of faces (
parser.add_argument("--scan-retinaface", type=int, help="When set to 1, scanning for additional faces will be performed using RetinaFace in a background thread, otherwise a simpler, faster face detection mechanism is used. When the maximum number of faces is 1, this option does nothing.", default=0)
parser.add_argument("--scan-every", type=int, help="Set after how many frames a scan for new faces should run", default=3)
parser.add_argument("--discard-after", type=int, help="Set the how long the tracker should keep looking for lost faces", default=10)
parser.add_argument("--max-feature-updates", type=int, help="When set to a value greater than zero, the internal minimum, maximum and median values of features are only updated this many times", default=0)
parser.add_argument("--no-3d-adapt", type=int, help="When set to 1, the 3D face model will not be adapted to increase the fit", default=0)
parser.add_argument("--video-out", help="Set this to the filename of an AVI file to save the tracking visualization as a video", default=None)
parser.add_argument("--raw-rgb", type=int, help="When this is set, raw RGB frames of the size given with \"-W\" and \"-H\" are read from standard input instead of reading a video", default=0)
parser.add_argument("--log-data", help="You can set a filename to which tracking data will be logged here", default="")
@ -130,7 +132,7 @@ try:
first = False
height, width, channels = frame.shape
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
tracker = Tracker(width, height, threshold=args.threshold, max_threads=args.max_threads, max_faces=args.faces, discard_after=args.discard_after, scan_every=args.scan_every, silent=False if args.silent == 0 else True, model_type=args.model, model_dir=args.model_dir, no_gaze=False if args.gaze_tracking != 0 else True, use_retinaface=args.scan_retinaface)
tracker = Tracker(width, height, threshold=args.threshold, max_threads=args.max_threads, max_faces=args.faces, discard_after=args.discard_after, scan_every=args.scan_every, silent=False if args.silent == 0 else True, model_type=args.model, model_dir=args.model_dir, no_gaze=False if args.gaze_tracking != 0 else True, use_retinaface=args.scan_retinaface, max_feature_updates=args.max_feature_updates, static_model=True if args.no_3d_adapt == 1 else False)
if not args.video_out is None:
out = cv2.VideoWriter(args.video_out, cv2.VideoWriter_fourcc('F','F','V','1'), 24, (width,height))

54
tracker.py
View file

@ -140,7 +140,7 @@ def worker_thread(session, frame, input, crop_info, queue, input_name, idx, trac
queue.put((session,))
class Feature():
def __init__(self, threshold=0.15, alpha=0.2, hard_factor=0.15, decay=0.001):
def __init__(self, threshold=0.15, alpha=0.2, hard_factor=0.15, decay=0.001, max_feature_updates=0):
self.median = remedian()
self.min = None
self.max = None
@ -151,40 +151,53 @@ class Feature():
self.hard_factor = hard_factor
self.decay = decay
self.last = 0
self.current_median = 0
self.update_count = 0
self.max_feature_updates = max_feature_updates
def update(self, x):
if self.max_feature_updates > 0:
self.update_count += 1
new = self.update_state(x)
filtered = self.last * self.alpha + new * (1 - self.alpha)
self.last = filtered
return filtered
def update_state(self, x):
updating = self.max_feature_updates < 1 or self.update_count < self.max_feature_updates
if updating:
self.median + x
median = self.median.median()
self.current_median = self.median.median()
median = self.current_median
if self.min is None:
if x < median and (median - x) / median > self.threshold:
if updating:
self.min = x
self.hard_min = self.min + self.hard_factor * (median - self.min)
return -1
return 0
else:
if x < self.min:
if updating:
self.min = x
self.hard_min = self.min + self.hard_factor * (median - self.min)
return -1
if self.max is None:
if x > median and (x - median) / median > self.threshold:
if updating:
self.max = x
self.hard_max = self.max - self.hard_factor * (self.max - median)
return 1
return 0
else:
if x > self.max:
if updating:
self.max = x
self.hard_max = self.max - self.hard_factor * (self.max - median)
return 1
if updating:
if self.min < self.hard_min:
self.min = self.hard_min * self.decay + self.min * (1 - self.decay)
if self.max > self.hard_max:
@ -198,21 +211,21 @@ class Feature():
return 0
class FeatureExtractor():
def __init__(self):
self.eye_l = Feature()
self.eye_r = Feature()
self.eyebrow_updown_l = Feature()
self.eyebrow_updown_r = Feature()
self.eyebrow_quirk_l = Feature(threshold=0.05)
self.eyebrow_quirk_r = Feature(threshold=0.05)
self.eyebrow_steepness_l = Feature(threshold=0.05)
self.eyebrow_steepness_r = Feature(threshold=0.05)
self.mouth_corner_updown_l = Feature()
self.mouth_corner_updown_r = Feature()
self.mouth_corner_inout_l = Feature(threshold=0.02)
self.mouth_corner_inout_r = Feature(threshold=0.02)
self.mouth_open = Feature()
self.mouth_wide = Feature(threshold=0.02)
def __init__(self, max_feature_updates=0):
self.eye_l = Feature(max_feature_updates=max_feature_updates)
self.eye_r = Feature(max_feature_updates=max_feature_updates)
self.eyebrow_updown_l = Feature(max_feature_updates=max_feature_updates)
self.eyebrow_updown_r = Feature(max_feature_updates=max_feature_updates)
self.eyebrow_quirk_l = Feature(threshold=0.05, max_feature_updates=max_feature_updates)
self.eyebrow_quirk_r = Feature(threshold=0.05, max_feature_updates=max_feature_updates)
self.eyebrow_steepness_l = Feature(threshold=0.05, max_feature_updates=max_feature_updates)
self.eyebrow_steepness_r = Feature(threshold=0.05, max_feature_updates=max_feature_updates)
self.mouth_corner_updown_l = Feature(max_feature_updates=max_feature_updates)
self.mouth_corner_updown_r = Feature(max_feature_updates=max_feature_updates)
self.mouth_corner_inout_l = Feature(threshold=0.02, max_feature_updates=max_feature_updates)
self.mouth_corner_inout_r = Feature(threshold=0.02, max_feature_updates=max_feature_updates)
self.mouth_open = Feature(max_feature_updates=max_feature_updates)
self.mouth_wide = Feature(threshold=0.02, max_feature_updates=max_feature_updates)
def align_points(self, a, b, pts):
a = tuple(a)
@ -314,7 +327,7 @@ class FaceInfo():
self.eye_blink = None
self.bbox = None
self.pnp_error = 0
self.features = FeatureExtractor()
self.features = FeatureExtractor(self.tracker.max_feature_updates)
self.current_features = {}
self.contour = np.zeros((21,3))
self.update_counts = np.zeros((66,2))
@ -354,6 +367,7 @@ class FaceInfo():
if self.conf < 0.4 or self.pnp_error > 300:
return
if not self.tracker.static_model:
max_runs = 1
eligible = np.delete(np.arange(0, 66), [30])
changed_any = False
@ -432,7 +446,7 @@ class FaceInfo():
class Tracker():
def __init__(self, width, height, model_type=3, threshold=0.6, max_faces=1, discard_after=5, scan_every=3, bbox_growth=0.0, max_threads=4, silent=False, model_dir=None, no_gaze=False, use_retinaface=False):
def __init__(self, width, height, model_type=3, threshold=0.6, max_faces=1, discard_after=5, scan_every=3, bbox_growth=0.0, max_threads=4, silent=False, model_dir=None, no_gaze=False, use_retinaface=False, max_feature_updates=0, static_model=False):
options = onnxruntime.SessionOptions()
options.inter_op_num_threads = 1
options.intra_op_num_threads = max(max_threads,4)
@ -593,6 +607,8 @@ class Tracker():
self.res_i = int(self.res)
self.no_gaze = no_gaze
self.debug_gaze = False
self.max_feature_updates = max_feature_updates
self.static_model = static_model
self.face_info = [FaceInfo(id, self) for id in range(max_faces)]
self.fail_count = 0