copilot/OpenSeeFace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

The basic 3D point model used for 3D point fitting is now based on data from fitting the points to head-pose-face-detection-female.mp4 over multiple iterations

Browse source
This commit is contained in:
Emiliana 2020-03-26 19:00:33 +01:00
parent 60fb3aac31
commit 4268abb00a
2 changed files with 136 additions and 76 deletions
Download patch file Download diff file Expand all files Collapse all files

63
facetracker.py
View file

@ -31,6 +31,8 @@ parser.add_argument("--model", type=int, help="This can be used to select the tr
parser.add_argument("--model-dir", help="This can be used to specify the path to the directory containing the .onnx model files", default=None)
parser.add_argument("--gaze-tracking", type=int, help="When set to 1, experimental blink detection and gaze tracking are enabled, which makes things slightly slower", default=1)
parser.add_argument("--face-id-offset", type=int, help="When set, this offset is added to all face ids, which can be useful for mixing tracking data from multiple network sources", default=0)
parser.add_argument("--repeat-video", type=int, help="When set to 1 and a video file was specified with -c, the tracker will loop the video until interrupted", default=0)
parser.add_argument("--dump-points", type=str, help="When set to a filename, the current face 3D points are made symmetric and dumped to the given file when quitting the visualization with the \"q\" key", default="")
args = parser.parse_args()
os.environ["OMP_NUM_THREADS"] = "1"
@ -54,7 +56,7 @@ import time
import cv2
import socket
import struct
from input_reader import InputReader, list_cameras
from input_reader import InputReader, VideoReader, list_cameras
from tracker import Tracker
target_ip = args.ip
@ -98,15 +100,27 @@ try:
target_duration = 0
if fps > 0:
target_duration = 1. / float(fps)
while input_reader.is_open():
repeat = args.repeat_video != 0 and type(input_reader.reader) == VideoReader
need_reinit = 0
while repeat or input_reader.is_open():
if not input_reader.is_open() or need_reinit == 1:
input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps)
need_reinit = 2
time.sleep(0.001)
continue
if not input_reader.is_ready():
time.sleep(0.001)
continue
ret, frame = input_reader.read()
if not ret:
if repeat:
if need_reinit == 0:
need_reinit = 1
continue
break
need_reinit = 0
frame_count += 1
now = time.time()
@ -237,6 +251,51 @@ try:
if args.visualize != 0:
cv2.imshow('OpenSeeFace Visualization', frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
if args.dump_points != "" and not faces is None and len(faces) > 0:
np.set_printoptions(threshold=sys.maxsize, precision=15)
pairs = [
(0, 16),
(1, 15),
(2, 14),
(3, 13),
(4, 12),
(5, 11),
(6, 10),
(7, 9),
(17, 26),
(18, 25),
(19, 24),
(20, 23),
(21, 22),
(31, 35),
(32, 34),
(36, 45),
(37, 44),
(38, 43),
(39, 42),
(40, 47),
(41, 46),
(48, 52),
(49, 51),
(56, 54),
(57, 53),
(58, 62),
(59, 61),
(65, 63)
]
points = copy.copy(faces[0].face_3d)
for a, b in pairs:
x = (points[a, 0] - points[b, 0]) / 2.0
y = (points[a, 1] + points[b, 1]) / 2.0
z = (points[a, 2] + points[b, 2]) / 2.0
points[a, 0] = x
points[b, 0] = -x
points[[a, b], 1] = y
points[[a, b], 2] = z
points[[8, 27, 28, 29, 33, 50, 55, 60, 64], 0] = 0.0
points[30, :] = 0.0
with open(args.dump_points, "w") as fh:
fh.write(repr(points))
break
duration = time.perf_counter() - frame_time

149
tracker.py
View file

@ -348,13 +348,14 @@ class FaceInfo():
return
max_runs = 1
eligible = np.arange(0, 66)
eligible = np.delete(np.arange(0, 66), [30])
changed_any = False
update_type = -1
d_o = np.ones((66,))
d_c = np.ones((66,))
for runs in range(max_runs):
r = 1.0 + np.random.random_sample((66,3)) * 0.02 - 0.01
r[30, :] = 1.0
if self.euler[0] > -165 and self.euler[0] < 145:
continue
elif self.euler[1] > -10 and self.euler[1] < 20:
@ -368,11 +369,11 @@ class FaceInfo():
elif self.euler[1] < -10:
update_type = 1
r[[0, 1, 2, 3, 4, 5, 6, 7, 17, 18, 19, 20, 21, 31, 32, 36, 37, 38, 39, 40, 41, 48, 49, 56, 57, 58, 59, 65], 2] = 1.0
eligible = [8, 9, 10, 11, 12, 13, 14, 15, 16, 22, 23, 24, 25, 26, 27, 28, 29, 30, 33, 34, 35, 42, 43, 44, 45, 46, 47, 50, 51, 52, 53, 54, 55, 60, 61, 62, 63, 64]
eligible = [8, 9, 10, 11, 12, 13, 14, 15, 16, 22, 23, 24, 25, 26, 27, 28, 29, 33, 34, 35, 42, 43, 44, 45, 46, 47, 50, 51, 52, 53, 54, 55, 60, 61, 62, 63, 64]
else:
update_type = 1
r[[9, 10, 11, 12, 13, 14, 15, 16, 22, 23, 24, 25, 26, 34, 35, 42, 43, 44, 45, 46, 47, 51, 52, 53, 54, 61, 62, 63], 2] = 1.0
eligible = [0, 1, 2, 3, 4, 5, 6, 7, 8, 17, 18, 19, 20, 21, 27, 28, 29, 30, 31, 32, 33, 36, 37, 38, 39, 40, 41, 48, 49, 50, 55, 56, 57, 58, 59, 60, 64, 65]
eligible = [0, 1, 2, 3, 4, 5, 6, 7, 8, 17, 18, 19, 20, 21, 27, 28, 29, 31, 32, 33, 36, 37, 38, 39, 40, 41, 48, 49, 50, 55, 56, 57, 58, 59, 60, 64, 65]
if self.limit_3d_adjustment:
eligible = np.nonzero(self.update_counts[:, update_type] < self.update_counts[:, abs(update_type - 1)] + self.update_count_delta)[0]
@ -489,78 +490,78 @@ class Tracker():
# PnP solving
self.face_3d = np.array([
[0.50194, 0.28030, -0.49561],
[0.48377, 0.15031, -0.50000],
[0.46233, 0.01987, -0.49781],
[0.43017, -0.11278, -0.49123],
[0.38130, -0.25112, -0.46491],
[0.31761, -0.35653, -0.40132],
[0.24840, -0.44335, -0.30482],
[0.15900, -0.51157, -0.22588],
[0.00194, -0.55490, -0.16009],
[-0.15900, -0.51157, -0.22588],
[-0.24840, -0.44335, -0.30482],
[-0.31761, -0.35653, -0.40132],
[-0.38130, -0.25112, -0.46491],
[-0.43017, -0.11278, -0.49123],
[-0.46233, 0.01987, -0.49781],
[-0.48377, 0.15031, -0.50000],
[-0.50194, 0.28030, -0.49561],
[0.44537, 0.38375, -0.22807],
[0.38759, 0.42826, -0.18421],
[0.32259, 0.44510, -0.12500],
[0.24917, 0.44510, -0.07895],
[0.19500, 0.42104, -0.06798],
[-0.19500, 0.42104, -0.06798],
[-0.24917, 0.44510, -0.07895],
[-0.32259, 0.44510, -0.12500],
[-0.38759, 0.42826, -0.18421],
[-0.44537, 0.38375, -0.22807],
[0.00000, 0.25503, -0.08553],
[0.00000, 0.16240, -0.04825],
[0.00000, 0.07218, -0.01535],
[0.00000, 0.00000, 0.0000],
[0.08353, -0.02165, -0.12281],
[0.04738, -0.05019, -0.08991],
[0.00000, -0.05895, -0.07456],
[-0.04738, -0.05019, -0.08991],
[-0.08353, -0.02165, -0.12281],
[0.32981, 0.28270, -0.19956],
[0.28528, 0.30435, -0.17544],
[0.23111, 0.29714, -0.16009],
[0.17213, 0.27789, -0.14474],
[0.22870, 0.24781, -0.16009],
[0.30093, 0.24661, -0.17544],
[-0.17213, 0.27789, -0.14474],
[-0.23111, 0.29714, -0.16009],
[-0.28528, 0.30435, -0.17544],
[-0.32981, 0.28270, -0.19956],
[-0.30093, 0.24661, -0.17544],
[-0.22870, 0.24781, -0.16009],
[0.10871, -0.14282, -0.16667],
[0.04809, -0.12118, -0.12500],
[0.00000, -0.14931, -0.10307],
[-0.04809, -0.12118, -0.12500],
[-0.10871, -0.14282, -0.16667],
[-0.11348, -0.24018, -0.16228],
[-0.05719, -0.28129, -0.14474],
[0.00000, -0.29644, -0.10965],
[0.05719, -0.28129, -0.14474],
[0.11348, -0.24018, -0.16228],
[0.14984, -0.16878, -0.19956],
[0.06736, -0.17922, -0.14254],
[0.00000, -0.18861, -0.10965],
[-0.06736, -0.17922, -0.14254],
[-0.14984, -0.16878, -0.19956],
[-0.04980, -0.21723, -0.14035],
[0.00000, -0.21201, -0.11404],
[0.04980, -0.21723, -0.14035],
[ 0.4551769692672 , 0.300895790030204, -0.764429433974752],
[ 0.448998827123556, 0.166995837790733, -0.765143004071253],
[ 0.437431554952677, 0.022655479179981, -0.739267175112735],
[ 0.415033422928434, -0.088941454648772, -0.747947437846473],
[ 0.389123587370091, -0.232380029794684, -0.704788385327458],
[ 0.334630113904382, -0.361265387599081, -0.615587579236862],
[ 0.263725112132858, -0.460009725616771, -0.491479221041573],
[ 0.16241621322721 , -0.558037146073869, -0.339445180872282],
[ 0. , -0.621079019321682, -0.287294770748887],
[-0.16241621322721 , -0.558037146073869, -0.339445180872282],
[-0.263725112132858, -0.460009725616771, -0.491479221041573],
[-0.334630113904382, -0.361265387599081, -0.615587579236862],
[-0.389123587370091, -0.232380029794684, -0.704788385327458],
[-0.415033422928434, -0.088941454648772, -0.747947437846473],
[-0.437431554952677, 0.022655479179981, -0.739267175112735],
[-0.448998827123556, 0.166995837790733, -0.765143004071253],
[-0.4551769692672 , 0.300895790030204, -0.764429433974752],
[ 0.385529968662985, 0.402800553948697, -0.310031082540741],
[ 0.322196658344302, 0.464439136821772, -0.250558059367669],
[ 0.25409760441282 , 0.46420381416882 , -0.208177722146526],
[ 0.186875436782135, 0.44706071961879 , -0.145299823706503],
[ 0.120880983543622, 0.423566314072968, -0.110757158774771],
[-0.120880983543622, 0.423566314072968, -0.110757158774771],
[-0.186875436782135, 0.44706071961879 , -0.145299823706503],
[-0.25409760441282 , 0.46420381416882 , -0.208177722146526],
[-0.322196658344302, 0.464439136821772, -0.250558059367669],
[-0.385529968662985, 0.402800553948697, -0.310031082540741],
[ 0. , 0.293332603215811, -0.137582088779393],
[ 0. , 0.194828701837823, -0.069158109325951],
[ 0. , 0.103844017393155, -0.009151819844964],
[ 0. , 0. , 0. ],
[ 0.080626352317973, -0.041276068128093, -0.134161035564826],
[ 0.046439347377934, -0.057675223874769, -0.102990627164664],
[ 0. , -0.068753126205604, -0.090545348482397],
[-0.046439347377934, -0.057675223874769, -0.102990627164664],
[-0.080626352317973, -0.041276068128093, -0.134161035564826],
[ 0.315905195966084, 0.298337502555443, -0.285107407636464],
[ 0.275252345439353, 0.312721904921771, -0.244558251170671],
[ 0.176394511553111, 0.311907184376107, -0.219205360345231],
[ 0.131229723798772, 0.284447361805627, -0.234239149487417],
[ 0.184124948330084, 0.260179585304867, -0.226590776513707],
[ 0.279433549294448, 0.267363071770222, -0.248441437111633],
[-0.131229723798772, 0.284447361805627, -0.234239149487417],
[-0.176394511553111, 0.311907184376107, -0.219205360345231],
[-0.275252345439353, 0.312721904921771, -0.244558251170671],
[-0.315905195966084, 0.298337502555443, -0.285107407636464],
[-0.279433549294448, 0.267363071770222, -0.248441437111633],
[-0.184124948330084, 0.260179585304867, -0.226590776513707],
[ 0.121155252430729, -0.208988660580347, -0.160606287940521],
[ 0.041356305910044, -0.194484199722098, -0.096159882202821],
[ 0. , -0.205180167345702, -0.083299217789729],
[-0.041356305910044, -0.194484199722098, -0.096159882202821],
[-0.121155252430729, -0.208988660580347, -0.160606287940521],
[-0.132325402795928, -0.290857984604968, -0.187067868218105],
[-0.064137791831655, -0.325377847425684, -0.158924039726607],
[ 0. , -0.343742581679188, -0.113925986025684],
[ 0.064137791831655, -0.325377847425684, -0.158924039726607],
[ 0.132325402795928, -0.290857984604968, -0.187067868218105],
[ 0.181481567104525, -0.243239316141725, -0.231284988892766],
[ 0.083999507750469, -0.239717753728704, -0.155256465640701],
[ 0. , -0.256058040176369, -0.0950619498899 ],
[-0.083999507750469, -0.239717753728704, -0.155256465640701],
[-0.181481567104525, -0.243239316141725, -0.231284988892766],
[-0.074036069749345, -0.250689938345682, -0.177346470406188],
[ 0. , -0.264945854681568, -0.112349967428413],
[ 0.074036069749345, -0.250689938345682, -0.177346470406188],
# Pupils and eyeball centers
[0.25799, 0.27608, -0.16923],
[-0.25799, 0.27608, -0.16923],
[0.25799, 0.27608, -0.24967],
[-0.25799, 0.27608, -0.24967],
], np.float32) * np.array([1.0, 1.0, 1.3])
[ 0.257990002632141, 0.276080012321472, -0.219998998939991],
[-0.257990002632141, 0.276080012321472, -0.219998998939991],
[ 0.257990002632141, 0.276080012321472, -0.324570998549461],
[-0.257990002632141, 0.276080012321472, -0.324570998549461]
], np.float32)
self.camera = np.array([[width, 0, width/2], [0, width, height/2], [0, 0, 1]], np.float32)
self.inverse_camera = np.linalg.inv(self.camera)