RetinaFace was added as a face detection scheme.

Licenses/Pytorch_Retinaface.txt

@@ -0,0 +1,21 @@
+MIT License 
+
+Copyright (c) 2019
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -141,6 +141,17 @@ For all training after the first model, a modified version of [Adaptive Wing Los
 
 For expression detection, [LIBSVM](https://www.csie.ntu.edu.tw/~cjlin/libsvm/) is used.
 
+Face detection is done using OpenCV's Haar Cascade classifier or RetinaFace.
+
+    @inproceedings{deng2019retinaface,
+      title={RetinaFace: Single-stage Dense Face Localisation in the Wild},
+      author={Deng, Jiankang and Guo, Jia and Yuxiang, Zhou and Jinke Yu and Irene Kotsia and Zafeiriou, Stefanos},
+      booktitle={arxiv},
+      year={2019}
+    }
+
+RetinaFace detection is based on [this](https://github.com/biubug6/Pytorch_Retinaface) implementation. The pretrained model was modified to remove unnecessary landmark detection and converted to ONNX format for a resolution of 384x384.
+
 # License
 
 The code and models are distributed under the BSD 2-clause license. 

facetracker.py

@@ -21,6 +21,7 @@ parser.add_argument("-v", "--visualize", type=int, help="Set this to 1 to visual
 parser.add_argument("-P", "--pnp-points", type=int, help="Set this to 1 to add the 3D fitting points to the visualization", default=0)
 parser.add_argument("-s", "--silent", type=int, help="Set this to 1 to prevent text output on the console", default=0)
 parser.add_argument("--faces", type=int, help="Set the maximum number of faces (slow)", default=1)
+parser.add_argument("--scan-retinaface", type=int, help="When set to 1, scanning for additional faces will be performed using RetinaFace in a background thread, otherwise a simpler, faster face detection mechanism is used. When the maximum number of faces is 1, this option does nothing.", default=0)
 parser.add_argument("--scan-every", type=int, help="Set after how many frames a scan for new faces should run", default=3)
 parser.add_argument("--discard-after", type=int, help="Set the how long the tracker should keep looking for lost faces", default=10)
 parser.add_argument("--video-out", help="Set this to the filename of an AVI file to save the tracking visualization as a video", default=None)
@@ -106,7 +107,7 @@ try:
             first = False
             height, width, channels = frame.shape
             sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-            tracker = Tracker(width, height, threshold=args.threshold, max_threads=args.max_threads, max_faces=args.faces, discard_after=args.discard_after, scan_every=args.scan_every, silent=False if args.silent == 0 else True, model_type=args.model, pnp_quality=args.high_quality_3d, model_dir=args.model_dir, no_gaze=False if args.gaze_tracking != 0 else True)
+            tracker = Tracker(width, height, threshold=args.threshold, max_threads=args.max_threads, max_faces=args.faces, discard_after=args.discard_after, scan_every=args.scan_every, silent=False if args.silent == 0 else True, model_type=args.model, pnp_quality=args.high_quality_3d, model_dir=args.model_dir, no_gaze=False if args.gaze_tracking != 0 else True, use_retinaface=args.scan_retinaface)
             if not args.video_out is None:
                 out = cv2.VideoWriter(args.video_out, cv2.VideoWriter_fourcc('F','F','V','1'), 24, (width,height))
 

retinaface.py

@@ -0,0 +1,150 @@
+import sys
+import numpy as np
+import cv2
+import onnxruntime
+import time
+import queue
+import threading
+import json
+import copy
+
+def py_cpu_nms(dets, thresh):
+    """ Pure Python NMS baseline.
+        Copyright (c) 2015 Microsoft
+        Licensed under The MIT License
+        Written by Ross Girshick
+    """
+    x1 = dets[:, 0]
+    y1 = dets[:, 1]
+    x2 = dets[:, 2]
+    y2 = dets[:, 3]
+    scores = dets[:, 4]
+
+    areas = (x2 - x1 + 1) * (y2 - y1 + 1)
+    order = scores.argsort()[::-1]
+
+    keep = []
+    while order.size > 0:
+        i = order[0]
+        keep.append(i)
+        xx1 = np.maximum(x1[i], x1[order[1:]])
+        yy1 = np.maximum(y1[i], y1[order[1:]])
+        xx2 = np.minimum(x2[i], x2[order[1:]])
+        yy2 = np.minimum(y2[i], y2[order[1:]])
+
+        w = np.maximum(0.0, xx2 - xx1 + 1)
+        h = np.maximum(0.0, yy2 - yy1 + 1)
+        inter = w * h
+        ovr = inter / (areas[i] + areas[order[1:]] - inter)
+
+        inds = np.where(ovr <= thresh)[0]
+        order = order[inds + 1]
+
+    return keep
+
+def decode(loc, priors, variances):
+    data = (
+        priors[:, :2] + loc[:, :2] * variances[0] * priors[:, 2:],
+        priors[:, 2:] * np.exp(loc[:, 2:] * variances[1])
+    )
+    boxes = np.concatenate(data, 1)
+    boxes[:, :2] -= boxes[:, 2:] / 2
+    boxes[:, 2:] += boxes[:, :2]
+    return boxes
+
+def worker_thread(rfd, frame):
+    results = rfd.detect_retina(frame, is_background=True)
+    rfd.results.put(results, False)
+    rfd.finished = True
+    rfd.running = False
+
+class RetinaFaceDetector():
+    def __init__(self, model_path="models/retinaface_opt.onnx", json_path="models/priorbox_384.json", threads=4, min_conf=0.3, nms_threshold=0.4, top_k=1):
+        options = onnxruntime.SessionOptions()
+        options.inter_op_num_threads = 1
+        options.intra_op_num_threads = threads
+        options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
+        options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
+        options.log_severity_level = 3
+        self.session = onnxruntime.InferenceSession(model_path, sess_options=options)
+        self.res = 384
+        with open(json_path, "r") as prior_file:
+            self.priorbox = np.array(json.loads(prior_file.read()))
+        self.min_conf = min_conf
+        self.nms_threshold = nms_threshold
+        self.top_k = top_k
+        self.finished = False
+        self.running = False
+        self.results = queue.Queue()
+
+    def detect_retina(self, frame, is_background=False):
+        h, w, _ = frame.shape
+        im = None
+        if h > w:
+            resize = h / self.res
+            im = cv2.resize(frame, (int(w / resize), self.res), interpolation=cv2.INTER_LINEAR)
+            resize = 1 / resize
+        else:
+            resize = w / self.res
+            im = cv2.resize(frame, (self.res, int(h / resize)), interpolation=cv2.INTER_LINEAR)
+            resize = 1 / resize
+        pad_h = self.res - im.shape[0]
+        pad_w = self.res - im.shape[1]
+        im = cv2.copyMakeBorder(im, 0, pad_h, 0, pad_w, cv2.BORDER_CONSTANT, (0, 0, 0))
+        im = np.float32(im)
+        scale = np.array((self.res, self.res, self.res, self.res))
+        im -= (104, 117, 123)
+        im = im.transpose(2, 0, 1)
+        im = np.expand_dims(im, 0)
+        output = self.session.run([], {"input0": im})
+        loc, conf = output[0][0], output[1][0]
+        boxes = decode(loc, self.priorbox, [0.1, 0.2])
+        boxes = boxes * scale / resize
+        scores = conf[:, 1]
+
+        inds = np.where(scores > self.min_conf)[0]
+        boxes = boxes[inds]
+        scores = scores[inds]
+
+        dets = np.hstack((boxes, scores[:, np.newaxis])).astype(np.float32, copy=False)
+        keep = py_cpu_nms(dets, self.nms_threshold)
+        dets = dets[keep, :]
+        dets = dets[:self.top_k, 0:4]
+        dets[:, 2:4] = dets[:, 2:4] - dets[:, 0:2]
+
+        if is_background:
+            upsize = dets[:, 2:4] * np.array([[0.1, 0.0]])
+            dets[:, 0:2] -= upsize
+            dets[:, 2:4] += upsize * 2
+
+        return list(map(tuple, dets))
+
+    def background_detect(self, frame):
+        if self.running or self.finished:
+            return
+        self.running = True
+        im = copy.copy(frame)
+        thread = threading.Thread(target=worker_thread, args=(self, im))
+        thread.start()
+
+    def get_results(self):
+        if self.finished:
+            results = []
+            try:
+                while True:
+                    detection = self.results.get(False)
+                    results.append(detection)
+            except:
+                "No error"
+            self.finished = False
+            return list(*results)
+        else:
+            return []
+
+if __name__== "__main__":
+    retina = RetinaFaceDetector(top_k=4)
+    im = cv2.imread(sys.argv[1], cv2.IMREAD_COLOR)
+    start = time.perf_counter()
+    faces = retina.detect_retina(im)
+    end = 1000 * (time.perf_counter() - start)
+    print(f"Runtime: {end:.3f}ms")

tracker.py

@@ -7,6 +7,7 @@ import time
 import queue
 import threading
 import copy
+from retinaface import RetinaFaceDetector
 
 def resolve(name):
     f = os.path.join(os.path.dirname(__file__), name)
@@ -226,7 +227,7 @@ class FaceInfo():
 
 
 class Tracker():
-    def __init__(self, width, height, model_type=3, threshold=0.4, max_faces=1, discard_after=5, scan_every=3, bbox_growth=0.0, max_threads=4, silent=False, pnp_quality=1, model_dir=None, no_gaze=False):
+    def __init__(self, width, height, model_type=3, threshold=0.4, max_faces=1, discard_after=5, scan_every=3, bbox_growth=0.0, max_threads=4, silent=False, pnp_quality=1, model_dir=None, no_gaze=False, use_retinaface=False):
         options = onnxruntime.SessionOptions()
         options.inter_op_num_threads = 1
         options.intra_op_num_threads = max_threads
@@ -248,6 +249,10 @@ class Tracker():
         else:
             model_base_path = model_dir
 
+        self.retinaface = RetinaFaceDetector(model_path=os.path.join(model_base_path, "retinaface_opt.onnx"), json_path=os.path.join(model_base_path, "priorbox_384.json"), threads=max_threads, top_k=max_faces)
+        self.retinaface_slow = RetinaFaceDetector(model_path=os.path.join(model_base_path, "retinaface_opt.onnx"), json_path=os.path.join(model_base_path, "priorbox_384.json"), threads=2, top_k=max_faces)
+        self.use_retinaface = use_retinaface
+
         # Single face instance with multiple threads
         self.session = onnxruntime.InferenceSession(os.path.join(model_base_path, model), sess_options=options)
 
@@ -664,13 +669,23 @@ class Tracker():
         new_faces.extend(self.faces)
         new_faces.extend(additional_faces)
         self.wait_count += 1
-        if self.detected < self.max_faces:
+        if self.detected == 0:
-            if self.detected == 0 or self.wait_count >= self.scan_every:
+            start_fd = time.perf_counter()
-                start_fd = time.perf_counter()
+            new_faces.extend(self.retinaface.detect_retina(frame))
-                gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+            duration_fd = 1000 * (time.perf_counter() - start_fd)
-                new_faces.extend(list(self.faceCascade.detectMultiScale(gray, 1.3, 4, 0 | cv2.CASCADE_SCALE_IMAGE, (50, 50))))
+            self.wait_count = 0
-                duration_fd = 1000 * (time.perf_counter() - start_fd)
+        elif self.detected < self.max_faces:
-                self.wait_count = 0
+            if self.use_retinaface > 0:
+                new_faces.extend(self.retinaface_slow.get_results())
+            if self.wait_count >= self.scan_every:
+                if self.use_retinaface > 0:
+                    self.retinaface_slow.background_detect(frame)
+                else:
+                    start_fd = time.perf_counter()
+                    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+                    new_faces.extend(list(self.faceCascade.detectMultiScale(gray, 1.3, 4, 0 | cv2.CASCADE_SCALE_IMAGE, (50, 50))))
+                    duration_fd = 1000 * (time.perf_counter() - start_fd)
+                    self.wait_count = 0
         else:
             self.wait_count = 0