copilot/OpenSeeFace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

The basic 3D point model used for 3D point fitting is now based on data from fitting the points to head-pose-face-detection-female.mp4 over multiple iterations

Browse source
This commit is contained in:
Emiliana 2020-03-26 19:00:33 +01:00
parent 60fb3aac31
commit 4268abb00a
2 changed files with 136 additions and 76 deletions
Download patch file Download diff file Expand all files Collapse all files

63
facetracker.py
View file

@ -31,6 +31,8 @@ parser.add_argument("--model", type=int, help="This can be used to select the tr
parser.add_argument("--model-dir", help="This can be used to specify the path to the directory containing the .onnx model files", default=None) parser.add_argument("--model-dir", help="This can be used to specify the path to the directory containing the .onnx model files", default=None)
parser.add_argument("--gaze-tracking", type=int, help="When set to 1, experimental blink detection and gaze tracking are enabled, which makes things slightly slower", default=1) parser.add_argument("--gaze-tracking", type=int, help="When set to 1, experimental blink detection and gaze tracking are enabled, which makes things slightly slower", default=1)
parser.add_argument("--face-id-offset", type=int, help="When set, this offset is added to all face ids, which can be useful for mixing tracking data from multiple network sources", default=0) parser.add_argument("--face-id-offset", type=int, help="When set, this offset is added to all face ids, which can be useful for mixing tracking data from multiple network sources", default=0)
parser.add_argument("--repeat-video", type=int, help="When set to 1 and a video file was specified with -c, the tracker will loop the video until interrupted", default=0)
parser.add_argument("--dump-points", type=str, help="When set to a filename, the current face 3D points are made symmetric and dumped to the given file when quitting the visualization with the \"q\" key", default="")
args = parser.parse_args() args = parser.parse_args()
os.environ["OMP_NUM_THREADS"] = "1" os.environ["OMP_NUM_THREADS"] = "1"
@ -54,7 +56,7 @@ import time
import cv2 import cv2
import socket import socket
import struct import struct
from input_reader import InputReader, list_cameras from input_reader import InputReader, VideoReader, list_cameras
from tracker import Tracker from tracker import Tracker
target_ip = args.ip target_ip = args.ip
@ -98,15 +100,27 @@ try:
target_duration = 0 target_duration = 0
if fps > 0: if fps > 0:
target_duration = 1. / float(fps) target_duration = 1. / float(fps)
while input_reader.is_open(): repeat = args.repeat_video != 0 and type(input_reader.reader) == VideoReader
need_reinit = 0
while repeat or input_reader.is_open():
if not input_reader.is_open() or need_reinit == 1:
input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps)
need_reinit = 2
time.sleep(0.001)
continue
if not input_reader.is_ready(): if not input_reader.is_ready():
time.sleep(0.001) time.sleep(0.001)
continue continue
ret, frame = input_reader.read() ret, frame = input_reader.read()
if not ret: if not ret:
if repeat:
if need_reinit == 0:
need_reinit = 1
continue
break break
need_reinit = 0
frame_count += 1 frame_count += 1
now = time.time() now = time.time()
@ -237,6 +251,51 @@ try:
if args.visualize != 0: if args.visualize != 0:
cv2.imshow('OpenSeeFace Visualization', frame) cv2.imshow('OpenSeeFace Visualization', frame)
if cv2.waitKey(1) & 0xFF == ord('q'): if cv2.waitKey(1) & 0xFF == ord('q'):
if args.dump_points != "" and not faces is None and len(faces) > 0:
np.set_printoptions(threshold=sys.maxsize, precision=15)
pairs = [
(0, 16),
(1, 15),
(2, 14),
(3, 13),
(4, 12),
(5, 11),
(6, 10),
(7, 9),
(17, 26),
(18, 25),
(19, 24),
(20, 23),
(21, 22),
(31, 35),
(32, 34),
(36, 45),
(37, 44),
(38, 43),
(39, 42),
(40, 47),
(41, 46),
(48, 52),
(49, 51),
(56, 54),
(57, 53),
(58, 62),
(59, 61),
(65, 63)
]
points = copy.copy(faces[0].face_3d)
for a, b in pairs:
x = (points[a, 0] - points[b, 0]) / 2.0
y = (points[a, 1] + points[b, 1]) / 2.0
z = (points[a, 2] + points[b, 2]) / 2.0
points[a, 0] = x
points[b, 0] = -x
points[[a, b], 1] = y
points[[a, b], 2] = z
points[[8, 27, 28, 29, 33, 50, 55, 60, 64], 0] = 0.0
points[30, :] = 0.0
with open(args.dump_points, "w") as fh:
fh.write(repr(points))
break break
duration = time.perf_counter() - frame_time duration = time.perf_counter() - frame_time

149
tracker.py
View file

@ -348,13 +348,14 @@ class FaceInfo():
return return
max_runs = 1 max_runs = 1
eligible = np.arange(0, 66) eligible = np.delete(np.arange(0, 66), [30])
changed_any = False changed_any = False
update_type = -1 update_type = -1
d_o = np.ones((66,)) d_o = np.ones((66,))
d_c = np.ones((66,)) d_c = np.ones((66,))
for runs in range(max_runs): for runs in range(max_runs):
r = 1.0 + np.random.random_sample((66,3)) * 0.02 - 0.01 r = 1.0 + np.random.random_sample((66,3)) * 0.02 - 0.01
r[30, :] = 1.0
if self.euler[0] > -165 and self.euler[0] < 145: if self.euler[0] > -165 and self.euler[0] < 145:
continue continue
elif self.euler[1] > -10 and self.euler[1] < 20: elif self.euler[1] > -10 and self.euler[1] < 20:
@ -368,11 +369,11 @@ class FaceInfo():
elif self.euler[1] < -10: elif self.euler[1] < -10:
update_type = 1 update_type = 1
r[[0, 1, 2, 3, 4, 5, 6, 7, 17, 18, 19, 20, 21, 31, 32, 36, 37, 38, 39, 40, 41, 48, 49, 56, 57, 58, 59, 65], 2] = 1.0 r[[0, 1, 2, 3, 4, 5, 6, 7, 17, 18, 19, 20, 21, 31, 32, 36, 37, 38, 39, 40, 41, 48, 49, 56, 57, 58, 59, 65], 2] = 1.0
eligible = [8, 9, 10, 11, 12, 13, 14, 15, 16, 22, 23, 24, 25, 26, 27, 28, 29, 30, 33, 34, 35, 42, 43, 44, 45, 46, 47, 50, 51, 52, 53, 54, 55, 60, 61, 62, 63, 64] eligible = [8, 9, 10, 11, 12, 13, 14, 15, 16, 22, 23, 24, 25, 26, 27, 28, 29, 33, 34, 35, 42, 43, 44, 45, 46, 47, 50, 51, 52, 53, 54, 55, 60, 61, 62, 63, 64]
else: else:
update_type = 1 update_type = 1
r[[9, 10, 11, 12, 13, 14, 15, 16, 22, 23, 24, 25, 26, 34, 35, 42, 43, 44, 45, 46, 47, 51, 52, 53, 54, 61, 62, 63], 2] = 1.0 r[[9, 10, 11, 12, 13, 14, 15, 16, 22, 23, 24, 25, 26, 34, 35, 42, 43, 44, 45, 46, 47, 51, 52, 53, 54, 61, 62, 63], 2] = 1.0
eligible = [0, 1, 2, 3, 4, 5, 6, 7, 8, 17, 18, 19, 20, 21, 27, 28, 29, 30, 31, 32, 33, 36, 37, 38, 39, 40, 41, 48, 49, 50, 55, 56, 57, 58, 59, 60, 64, 65] eligible = [0, 1, 2, 3, 4, 5, 6, 7, 8, 17, 18, 19, 20, 21, 27, 28, 29, 31, 32, 33, 36, 37, 38, 39, 40, 41, 48, 49, 50, 55, 56, 57, 58, 59, 60, 64, 65]
if self.limit_3d_adjustment: if self.limit_3d_adjustment:
eligible = np.nonzero(self.update_counts[:, update_type] < self.update_counts[:, abs(update_type - 1)] + self.update_count_delta)[0] eligible = np.nonzero(self.update_counts[:, update_type] < self.update_counts[:, abs(update_type - 1)] + self.update_count_delta)[0]
@ -489,78 +490,78 @@ class Tracker():
# PnP solving # PnP solving
self.face_3d = np.array([ self.face_3d = np.array([
[0.50194, 0.28030, -0.49561], [ 0.4551769692672 , 0.300895790030204, -0.764429433974752],
[0.48377, 0.15031, -0.50000], [ 0.448998827123556, 0.166995837790733, -0.765143004071253],
[0.46233, 0.01987, -0.49781], [ 0.437431554952677, 0.022655479179981, -0.739267175112735],
[0.43017, -0.11278, -0.49123], [ 0.415033422928434, -0.088941454648772, -0.747947437846473],
[0.38130, -0.25112, -0.46491], [ 0.389123587370091, -0.232380029794684, -0.704788385327458],
[0.31761, -0.35653, -0.40132], [ 0.334630113904382, -0.361265387599081, -0.615587579236862],
[0.24840, -0.44335, -0.30482], [ 0.263725112132858, -0.460009725616771, -0.491479221041573],
[0.15900, -0.51157, -0.22588], [ 0.16241621322721 , -0.558037146073869, -0.339445180872282],
[0.00194, -0.55490, -0.16009], [ 0. , -0.621079019321682, -0.287294770748887],
[-0.15900, -0.51157, -0.22588], [-0.16241621322721 , -0.558037146073869, -0.339445180872282],
[-0.24840, -0.44335, -0.30482], [-0.263725112132858, -0.460009725616771, -0.491479221041573],
[-0.31761, -0.35653, -0.40132], [-0.334630113904382, -0.361265387599081, -0.615587579236862],
[-0.38130, -0.25112, -0.46491], [-0.389123587370091, -0.232380029794684, -0.704788385327458],
[-0.43017, -0.11278, -0.49123], [-0.415033422928434, -0.088941454648772, -0.747947437846473],
[-0.46233, 0.01987, -0.49781], [-0.437431554952677, 0.022655479179981, -0.739267175112735],
[-0.48377, 0.15031, -0.50000], [-0.448998827123556, 0.166995837790733, -0.765143004071253],
[-0.50194, 0.28030, -0.49561], [-0.4551769692672 , 0.300895790030204, -0.764429433974752],
[0.44537, 0.38375, -0.22807], [ 0.385529968662985, 0.402800553948697, -0.310031082540741],
[0.38759, 0.42826, -0.18421], [ 0.322196658344302, 0.464439136821772, -0.250558059367669],
[0.32259, 0.44510, -0.12500], [ 0.25409760441282 , 0.46420381416882 , -0.208177722146526],
[0.24917, 0.44510, -0.07895], [ 0.186875436782135, 0.44706071961879 , -0.145299823706503],
[0.19500, 0.42104, -0.06798], [ 0.120880983543622, 0.423566314072968, -0.110757158774771],
[-0.19500, 0.42104, -0.06798], [-0.120880983543622, 0.423566314072968, -0.110757158774771],
[-0.24917, 0.44510, -0.07895], [-0.186875436782135, 0.44706071961879 , -0.145299823706503],
[-0.32259, 0.44510, -0.12500], [-0.25409760441282 , 0.46420381416882 , -0.208177722146526],
[-0.38759, 0.42826, -0.18421], [-0.322196658344302, 0.464439136821772, -0.250558059367669],
[-0.44537, 0.38375, -0.22807], [-0.385529968662985, 0.402800553948697, -0.310031082540741],
[0.00000, 0.25503, -0.08553], [ 0. , 0.293332603215811, -0.137582088779393],
[0.00000, 0.16240, -0.04825], [ 0. , 0.194828701837823, -0.069158109325951],
[0.00000, 0.07218, -0.01535], [ 0. , 0.103844017393155, -0.009151819844964],
[0.00000, 0.00000, 0.0000], [ 0. , 0. , 0. ],
[0.08353, -0.02165, -0.12281], [ 0.080626352317973, -0.041276068128093, -0.134161035564826],
[0.04738, -0.05019, -0.08991], [ 0.046439347377934, -0.057675223874769, -0.102990627164664],
[0.00000, -0.05895, -0.07456], [ 0. , -0.068753126205604, -0.090545348482397],
[-0.04738, -0.05019, -0.08991], [-0.046439347377934, -0.057675223874769, -0.102990627164664],
[-0.08353, -0.02165, -0.12281], [-0.080626352317973, -0.041276068128093, -0.134161035564826],
[0.32981, 0.28270, -0.19956], [ 0.315905195966084, 0.298337502555443, -0.285107407636464],
[0.28528, 0.30435, -0.17544], [ 0.275252345439353, 0.312721904921771, -0.244558251170671],
[0.23111, 0.29714, -0.16009], [ 0.176394511553111, 0.311907184376107, -0.219205360345231],
[0.17213, 0.27789, -0.14474], [ 0.131229723798772, 0.284447361805627, -0.234239149487417],
[0.22870, 0.24781, -0.16009], [ 0.184124948330084, 0.260179585304867, -0.226590776513707],
[0.30093, 0.24661, -0.17544], [ 0.279433549294448, 0.267363071770222, -0.248441437111633],
[-0.17213, 0.27789, -0.14474], [-0.131229723798772, 0.284447361805627, -0.234239149487417],
[-0.23111, 0.29714, -0.16009], [-0.176394511553111, 0.311907184376107, -0.219205360345231],
[-0.28528, 0.30435, -0.17544], [-0.275252345439353, 0.312721904921771, -0.244558251170671],
[-0.32981, 0.28270, -0.19956], [-0.315905195966084, 0.298337502555443, -0.285107407636464],
[-0.30093, 0.24661, -0.17544], [-0.279433549294448, 0.267363071770222, -0.248441437111633],
[-0.22870, 0.24781, -0.16009], [-0.184124948330084, 0.260179585304867, -0.226590776513707],
[0.10871, -0.14282, -0.16667], [ 0.121155252430729, -0.208988660580347, -0.160606287940521],
[0.04809, -0.12118, -0.12500], [ 0.041356305910044, -0.194484199722098, -0.096159882202821],
[0.00000, -0.14931, -0.10307], [ 0. , -0.205180167345702, -0.083299217789729],
[-0.04809, -0.12118, -0.12500], [-0.041356305910044, -0.194484199722098, -0.096159882202821],
[-0.10871, -0.14282, -0.16667], [-0.121155252430729, -0.208988660580347, -0.160606287940521],
[-0.11348, -0.24018, -0.16228], [-0.132325402795928, -0.290857984604968, -0.187067868218105],
[-0.05719, -0.28129, -0.14474], [-0.064137791831655, -0.325377847425684, -0.158924039726607],
[0.00000, -0.29644, -0.10965], [ 0. , -0.343742581679188, -0.113925986025684],
[0.05719, -0.28129, -0.14474], [ 0.064137791831655, -0.325377847425684, -0.158924039726607],
[0.11348, -0.24018, -0.16228], [ 0.132325402795928, -0.290857984604968, -0.187067868218105],
[0.14984, -0.16878, -0.19956], [ 0.181481567104525, -0.243239316141725, -0.231284988892766],
[0.06736, -0.17922, -0.14254], [ 0.083999507750469, -0.239717753728704, -0.155256465640701],
[0.00000, -0.18861, -0.10965], [ 0. , -0.256058040176369, -0.0950619498899 ],
[-0.06736, -0.17922, -0.14254], [-0.083999507750469, -0.239717753728704, -0.155256465640701],
[-0.14984, -0.16878, -0.19956], [-0.181481567104525, -0.243239316141725, -0.231284988892766],
[-0.04980, -0.21723, -0.14035], [-0.074036069749345, -0.250689938345682, -0.177346470406188],
[0.00000, -0.21201, -0.11404], [ 0. , -0.264945854681568, -0.112349967428413],
[0.04980, -0.21723, -0.14035], [ 0.074036069749345, -0.250689938345682, -0.177346470406188],
# Pupils and eyeball centers # Pupils and eyeball centers
[0.25799, 0.27608, -0.16923], [ 0.257990002632141, 0.276080012321472, -0.219998998939991],
[-0.25799, 0.27608, -0.16923], [-0.257990002632141, 0.276080012321472, -0.219998998939991],
[0.25799, 0.27608, -0.24967], [ 0.257990002632141, 0.276080012321472, -0.324570998549461],
[-0.25799, 0.27608, -0.24967], [-0.257990002632141, 0.276080012321472, -0.324570998549461]
], np.float32) * np.array([1.0, 1.0, 1.3]) ], np.float32)
self.camera = np.array([[width, 0, width/2], [0, width, height/2], [0, 0, 1]], np.float32) self.camera = np.array([[width, 0, width/2], [0, width, height/2], [0, 0, 1]], np.float32)
self.inverse_camera = np.linalg.inv(self.camera) self.inverse_camera = np.linalg.inv(self.camera)