diff --git a/facetracker.py b/facetracker.py index ff538f1..ce36c92 100644 --- a/facetracker.py +++ b/facetracker.py @@ -31,6 +31,8 @@ parser.add_argument("--model", type=int, help="This can be used to select the tr parser.add_argument("--model-dir", help="This can be used to specify the path to the directory containing the .onnx model files", default=None) parser.add_argument("--gaze-tracking", type=int, help="When set to 1, experimental blink detection and gaze tracking are enabled, which makes things slightly slower", default=1) parser.add_argument("--face-id-offset", type=int, help="When set, this offset is added to all face ids, which can be useful for mixing tracking data from multiple network sources", default=0) +parser.add_argument("--repeat-video", type=int, help="When set to 1 and a video file was specified with -c, the tracker will loop the video until interrupted", default=0) +parser.add_argument("--dump-points", type=str, help="When set to a filename, the current face 3D points are made symmetric and dumped to the given file when quitting the visualization with the \"q\" key", default="") args = parser.parse_args() os.environ["OMP_NUM_THREADS"] = "1" @@ -54,7 +56,7 @@ import time import cv2 import socket import struct -from input_reader import InputReader, list_cameras +from input_reader import InputReader, VideoReader, list_cameras from tracker import Tracker target_ip = args.ip @@ -98,15 +100,27 @@ try: target_duration = 0 if fps > 0: target_duration = 1. / float(fps) - while input_reader.is_open(): + repeat = args.repeat_video != 0 and type(input_reader.reader) == VideoReader + need_reinit = 0 + while repeat or input_reader.is_open(): + if not input_reader.is_open() or need_reinit == 1: + input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps) + need_reinit = 2 + time.sleep(0.001) + continue if not input_reader.is_ready(): time.sleep(0.001) continue ret, frame = input_reader.read() if not ret: + if repeat: + if need_reinit == 0: + need_reinit = 1 + continue break + need_reinit = 0 frame_count += 1 now = time.time() @@ -237,6 +251,51 @@ try: if args.visualize != 0: cv2.imshow('OpenSeeFace Visualization', frame) if cv2.waitKey(1) & 0xFF == ord('q'): + if args.dump_points != "" and not faces is None and len(faces) > 0: + np.set_printoptions(threshold=sys.maxsize, precision=15) + pairs = [ + (0, 16), + (1, 15), + (2, 14), + (3, 13), + (4, 12), + (5, 11), + (6, 10), + (7, 9), + (17, 26), + (18, 25), + (19, 24), + (20, 23), + (21, 22), + (31, 35), + (32, 34), + (36, 45), + (37, 44), + (38, 43), + (39, 42), + (40, 47), + (41, 46), + (48, 52), + (49, 51), + (56, 54), + (57, 53), + (58, 62), + (59, 61), + (65, 63) + ] + points = copy.copy(faces[0].face_3d) + for a, b in pairs: + x = (points[a, 0] - points[b, 0]) / 2.0 + y = (points[a, 1] + points[b, 1]) / 2.0 + z = (points[a, 2] + points[b, 2]) / 2.0 + points[a, 0] = x + points[b, 0] = -x + points[[a, b], 1] = y + points[[a, b], 2] = z + points[[8, 27, 28, 29, 33, 50, 55, 60, 64], 0] = 0.0 + points[30, :] = 0.0 + with open(args.dump_points, "w") as fh: + fh.write(repr(points)) break duration = time.perf_counter() - frame_time diff --git a/tracker.py b/tracker.py index 21cfc1b..e17aa03 100644 --- a/tracker.py +++ b/tracker.py @@ -348,13 +348,14 @@ class FaceInfo(): return max_runs = 1 - eligible = np.arange(0, 66) + eligible = np.delete(np.arange(0, 66), [30]) changed_any = False update_type = -1 d_o = np.ones((66,)) d_c = np.ones((66,)) for runs in range(max_runs): r = 1.0 + np.random.random_sample((66,3)) * 0.02 - 0.01 + r[30, :] = 1.0 if self.euler[0] > -165 and self.euler[0] < 145: continue elif self.euler[1] > -10 and self.euler[1] < 20: @@ -368,11 +369,11 @@ class FaceInfo(): elif self.euler[1] < -10: update_type = 1 r[[0, 1, 2, 3, 4, 5, 6, 7, 17, 18, 19, 20, 21, 31, 32, 36, 37, 38, 39, 40, 41, 48, 49, 56, 57, 58, 59, 65], 2] = 1.0 - eligible = [8, 9, 10, 11, 12, 13, 14, 15, 16, 22, 23, 24, 25, 26, 27, 28, 29, 30, 33, 34, 35, 42, 43, 44, 45, 46, 47, 50, 51, 52, 53, 54, 55, 60, 61, 62, 63, 64] + eligible = [8, 9, 10, 11, 12, 13, 14, 15, 16, 22, 23, 24, 25, 26, 27, 28, 29, 33, 34, 35, 42, 43, 44, 45, 46, 47, 50, 51, 52, 53, 54, 55, 60, 61, 62, 63, 64] else: update_type = 1 r[[9, 10, 11, 12, 13, 14, 15, 16, 22, 23, 24, 25, 26, 34, 35, 42, 43, 44, 45, 46, 47, 51, 52, 53, 54, 61, 62, 63], 2] = 1.0 - eligible = [0, 1, 2, 3, 4, 5, 6, 7, 8, 17, 18, 19, 20, 21, 27, 28, 29, 30, 31, 32, 33, 36, 37, 38, 39, 40, 41, 48, 49, 50, 55, 56, 57, 58, 59, 60, 64, 65] + eligible = [0, 1, 2, 3, 4, 5, 6, 7, 8, 17, 18, 19, 20, 21, 27, 28, 29, 31, 32, 33, 36, 37, 38, 39, 40, 41, 48, 49, 50, 55, 56, 57, 58, 59, 60, 64, 65] if self.limit_3d_adjustment: eligible = np.nonzero(self.update_counts[:, update_type] < self.update_counts[:, abs(update_type - 1)] + self.update_count_delta)[0] @@ -489,78 +490,78 @@ class Tracker(): # PnP solving self.face_3d = np.array([ - [0.50194, 0.28030, -0.49561], - [0.48377, 0.15031, -0.50000], - [0.46233, 0.01987, -0.49781], - [0.43017, -0.11278, -0.49123], - [0.38130, -0.25112, -0.46491], - [0.31761, -0.35653, -0.40132], - [0.24840, -0.44335, -0.30482], - [0.15900, -0.51157, -0.22588], - [0.00194, -0.55490, -0.16009], - [-0.15900, -0.51157, -0.22588], - [-0.24840, -0.44335, -0.30482], - [-0.31761, -0.35653, -0.40132], - [-0.38130, -0.25112, -0.46491], - [-0.43017, -0.11278, -0.49123], - [-0.46233, 0.01987, -0.49781], - [-0.48377, 0.15031, -0.50000], - [-0.50194, 0.28030, -0.49561], - [0.44537, 0.38375, -0.22807], - [0.38759, 0.42826, -0.18421], - [0.32259, 0.44510, -0.12500], - [0.24917, 0.44510, -0.07895], - [0.19500, 0.42104, -0.06798], - [-0.19500, 0.42104, -0.06798], - [-0.24917, 0.44510, -0.07895], - [-0.32259, 0.44510, -0.12500], - [-0.38759, 0.42826, -0.18421], - [-0.44537, 0.38375, -0.22807], - [0.00000, 0.25503, -0.08553], - [0.00000, 0.16240, -0.04825], - [0.00000, 0.07218, -0.01535], - [0.00000, 0.00000, 0.0000], - [0.08353, -0.02165, -0.12281], - [0.04738, -0.05019, -0.08991], - [0.00000, -0.05895, -0.07456], - [-0.04738, -0.05019, -0.08991], - [-0.08353, -0.02165, -0.12281], - [0.32981, 0.28270, -0.19956], - [0.28528, 0.30435, -0.17544], - [0.23111, 0.29714, -0.16009], - [0.17213, 0.27789, -0.14474], - [0.22870, 0.24781, -0.16009], - [0.30093, 0.24661, -0.17544], - [-0.17213, 0.27789, -0.14474], - [-0.23111, 0.29714, -0.16009], - [-0.28528, 0.30435, -0.17544], - [-0.32981, 0.28270, -0.19956], - [-0.30093, 0.24661, -0.17544], - [-0.22870, 0.24781, -0.16009], - [0.10871, -0.14282, -0.16667], - [0.04809, -0.12118, -0.12500], - [0.00000, -0.14931, -0.10307], - [-0.04809, -0.12118, -0.12500], - [-0.10871, -0.14282, -0.16667], - [-0.11348, -0.24018, -0.16228], - [-0.05719, -0.28129, -0.14474], - [0.00000, -0.29644, -0.10965], - [0.05719, -0.28129, -0.14474], - [0.11348, -0.24018, -0.16228], - [0.14984, -0.16878, -0.19956], - [0.06736, -0.17922, -0.14254], - [0.00000, -0.18861, -0.10965], - [-0.06736, -0.17922, -0.14254], - [-0.14984, -0.16878, -0.19956], - [-0.04980, -0.21723, -0.14035], - [0.00000, -0.21201, -0.11404], - [0.04980, -0.21723, -0.14035], + [ 0.4551769692672 , 0.300895790030204, -0.764429433974752], + [ 0.448998827123556, 0.166995837790733, -0.765143004071253], + [ 0.437431554952677, 0.022655479179981, -0.739267175112735], + [ 0.415033422928434, -0.088941454648772, -0.747947437846473], + [ 0.389123587370091, -0.232380029794684, -0.704788385327458], + [ 0.334630113904382, -0.361265387599081, -0.615587579236862], + [ 0.263725112132858, -0.460009725616771, -0.491479221041573], + [ 0.16241621322721 , -0.558037146073869, -0.339445180872282], + [ 0. , -0.621079019321682, -0.287294770748887], + [-0.16241621322721 , -0.558037146073869, -0.339445180872282], + [-0.263725112132858, -0.460009725616771, -0.491479221041573], + [-0.334630113904382, -0.361265387599081, -0.615587579236862], + [-0.389123587370091, -0.232380029794684, -0.704788385327458], + [-0.415033422928434, -0.088941454648772, -0.747947437846473], + [-0.437431554952677, 0.022655479179981, -0.739267175112735], + [-0.448998827123556, 0.166995837790733, -0.765143004071253], + [-0.4551769692672 , 0.300895790030204, -0.764429433974752], + [ 0.385529968662985, 0.402800553948697, -0.310031082540741], + [ 0.322196658344302, 0.464439136821772, -0.250558059367669], + [ 0.25409760441282 , 0.46420381416882 , -0.208177722146526], + [ 0.186875436782135, 0.44706071961879 , -0.145299823706503], + [ 0.120880983543622, 0.423566314072968, -0.110757158774771], + [-0.120880983543622, 0.423566314072968, -0.110757158774771], + [-0.186875436782135, 0.44706071961879 , -0.145299823706503], + [-0.25409760441282 , 0.46420381416882 , -0.208177722146526], + [-0.322196658344302, 0.464439136821772, -0.250558059367669], + [-0.385529968662985, 0.402800553948697, -0.310031082540741], + [ 0. , 0.293332603215811, -0.137582088779393], + [ 0. , 0.194828701837823, -0.069158109325951], + [ 0. , 0.103844017393155, -0.009151819844964], + [ 0. , 0. , 0. ], + [ 0.080626352317973, -0.041276068128093, -0.134161035564826], + [ 0.046439347377934, -0.057675223874769, -0.102990627164664], + [ 0. , -0.068753126205604, -0.090545348482397], + [-0.046439347377934, -0.057675223874769, -0.102990627164664], + [-0.080626352317973, -0.041276068128093, -0.134161035564826], + [ 0.315905195966084, 0.298337502555443, -0.285107407636464], + [ 0.275252345439353, 0.312721904921771, -0.244558251170671], + [ 0.176394511553111, 0.311907184376107, -0.219205360345231], + [ 0.131229723798772, 0.284447361805627, -0.234239149487417], + [ 0.184124948330084, 0.260179585304867, -0.226590776513707], + [ 0.279433549294448, 0.267363071770222, -0.248441437111633], + [-0.131229723798772, 0.284447361805627, -0.234239149487417], + [-0.176394511553111, 0.311907184376107, -0.219205360345231], + [-0.275252345439353, 0.312721904921771, -0.244558251170671], + [-0.315905195966084, 0.298337502555443, -0.285107407636464], + [-0.279433549294448, 0.267363071770222, -0.248441437111633], + [-0.184124948330084, 0.260179585304867, -0.226590776513707], + [ 0.121155252430729, -0.208988660580347, -0.160606287940521], + [ 0.041356305910044, -0.194484199722098, -0.096159882202821], + [ 0. , -0.205180167345702, -0.083299217789729], + [-0.041356305910044, -0.194484199722098, -0.096159882202821], + [-0.121155252430729, -0.208988660580347, -0.160606287940521], + [-0.132325402795928, -0.290857984604968, -0.187067868218105], + [-0.064137791831655, -0.325377847425684, -0.158924039726607], + [ 0. , -0.343742581679188, -0.113925986025684], + [ 0.064137791831655, -0.325377847425684, -0.158924039726607], + [ 0.132325402795928, -0.290857984604968, -0.187067868218105], + [ 0.181481567104525, -0.243239316141725, -0.231284988892766], + [ 0.083999507750469, -0.239717753728704, -0.155256465640701], + [ 0. , -0.256058040176369, -0.0950619498899 ], + [-0.083999507750469, -0.239717753728704, -0.155256465640701], + [-0.181481567104525, -0.243239316141725, -0.231284988892766], + [-0.074036069749345, -0.250689938345682, -0.177346470406188], + [ 0. , -0.264945854681568, -0.112349967428413], + [ 0.074036069749345, -0.250689938345682, -0.177346470406188], # Pupils and eyeball centers - [0.25799, 0.27608, -0.16923], - [-0.25799, 0.27608, -0.16923], - [0.25799, 0.27608, -0.24967], - [-0.25799, 0.27608, -0.24967], - ], np.float32) * np.array([1.0, 1.0, 1.3]) + [ 0.257990002632141, 0.276080012321472, -0.219998998939991], + [-0.257990002632141, 0.276080012321472, -0.219998998939991], + [ 0.257990002632141, 0.276080012321472, -0.324570998549461], + [-0.257990002632141, 0.276080012321472, -0.324570998549461] + ], np.float32) self.camera = np.array([[width, 0, width/2], [0, width, height/2], [0, 0, 1]], np.float32) self.inverse_camera = np.linalg.inv(self.camera)