copilot/OpenSeeFace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' of https://github.com/qt-autumn/OpenSeeFace

Browse source
This commit is contained in:
Autumn 2021-10-01 02:35:34 +10:00
commit d89e8dacda
5 changed files with 14 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -2,7 +2,7 @@
# Overview # Overview
**Note**: This is a tracking library, **not** a stand-alone avatar puppeteering program. I'm also working on [VSeeFace](https://www.vseeface.icu/), which allows animating [VRM](https://vrm.dev/en/how_to_make_vrm/) and [VSFAvatar](https://www.youtube.com/watch?v=jhQ8DF87I5I) 3D models by using OpenSeeFace tracking. [VTube Studio](https://denchisoft.com/) uses OpenSeeFace for webcam based tracking to animate Live2D models. **Note**: This is a tracking library, **not** a stand-alone avatar puppeteering program. I'm also working on [VSeeFace](https://www.vseeface.icu/), which allows animating [VRM](https://vrm.dev/en/how_to_make_vrm/) and [VSFAvatar](https://www.youtube.com/watch?v=jhQ8DF87I5I) 3D models by using OpenSeeFace tracking. [VTube Studio](https://denchisoft.com/) uses OpenSeeFace for webcam based tracking to animate Live2D models. A renderer for the Godot engine can be found [here](https://github.com/you-win/openseeface-gd).
This project implements a facial landmark detection model based on MobileNetV3. This project implements a facial landmark detection model based on MobileNetV3.

6
dshowcapture.py
View file

@ -104,7 +104,7 @@ class DShowCapture():
def get_device(self, device_number): def get_device(self, device_number):
self.lib.get_device(self.cap, device_number, self.name_buffer, 255) self.lib.get_device(self.cap, device_number, self.name_buffer, 255)
name_str = str(self.name_buffer.value.decode('utf8')) name_str = str(self.name_buffer.value.decode('utf8', 'surrogateescape'))
return name_str return name_str
def get_info(self): def get_info(self):
@ -113,7 +113,7 @@ class DShowCapture():
json_length = self.lib.get_json_length(self.cap); json_length = self.lib.get_json_length(self.cap);
json_buffer = create_string_buffer(json_length) json_buffer = create_string_buffer(json_length)
self.lib.get_json(self.cap, json_buffer, json_length); self.lib.get_json(self.cap, json_buffer, json_length);
json_text = str(json_buffer.value.decode('utf8')) json_text = str(json_buffer.value.decode('utf8', 'surrogateescape'))
self.info = json.loads(json_text) self.info = json.loads(json_text)
for cam in self.info: for cam in self.info:
cam["type"] = "DirectShow" cam["type"] = "DirectShow"
@ -124,7 +124,7 @@ class DShowCapture():
json_length = self.bm_lib.get_json_length(); json_length = self.bm_lib.get_json_length();
json_buffer = create_string_buffer(json_length) json_buffer = create_string_buffer(json_length)
self.bm_lib.get_json(json_buffer, json_length); self.bm_lib.get_json(json_buffer, json_length);
json_text = str(json_buffer.value.decode('utf8')) json_text = str(json_buffer.value.decode('utf8', 'surrogateescape'))
bm_info = json.loads(json_text) bm_info = json.loads(json_text)
dshow_len = len(self.info) dshow_len = len(self.info)
for cam in bm_info: for cam in bm_info:

6
facetracker.py
View file

@ -292,7 +292,7 @@ try:
packet.extend(bytearray(struct.pack("f", f.translation[1]))) packet.extend(bytearray(struct.pack("f", f.translation[1])))
packet.extend(bytearray(struct.pack("f", f.translation[2]))) packet.extend(bytearray(struct.pack("f", f.translation[2])))
if not log is None: if not log is None:
log.write(f"{frame_count},{now},{width},{height},{args.fps},{face_num},{f.id},{f.eye_blink[0]},{f.eye_blink[1]},{f.conf},{f.success},{f.pnp_error},{f.quaternion[0]},{f.quaternion[1]},{f.quaternion[2]},{f.quaternion[3]},{f.euler[0]},{f.euler[1]},{f.euler[2]},{f.rotation[0]},{f.rotation[1]},{f.rotation[2]},{f.translation[0]},{f.translation[1]},{f.translation[2]}") log.write(f"{frame_count},{now},{width},{height},{fps},{face_num},{f.id},{f.eye_blink[0]},{f.eye_blink[1]},{f.conf},{f.success},{f.pnp_error},{f.quaternion[0]},{f.quaternion[1]},{f.quaternion[2]},{f.quaternion[3]},{f.euler[0]},{f.euler[1]},{f.euler[2]},{f.rotation[0]},{f.rotation[1]},{f.rotation[2]},{f.translation[0]},{f.translation[1]},{f.translation[2]}")
for (x,y,c) in f.lms: for (x,y,c) in f.lms:
packet.extend(bytearray(struct.pack("f", c))) packet.extend(bytearray(struct.pack("f", c)))
if args.visualize > 1: if args.visualize > 1:
@ -304,9 +304,9 @@ try:
packet.extend(bytearray(struct.pack("f", x))) packet.extend(bytearray(struct.pack("f", x)))
if not log is None: if not log is None:
log.write(f",{y},{x},{c}") log.write(f",{y},{x},{c}")
if pt_num == 66 and (f.eye_blink[0] < 0.30 or c < 0.30): if pt_num == 66 and (f.eye_blink[0] < 0.30 or c < 0.20):
continue continue
if pt_num == 67 and (f.eye_blink[1] < 0.30 or c < 0.30): if pt_num == 67 and (f.eye_blink[1] < 0.30 or c < 0.20):
continue continue
x = int(x + 0.5) x = int(x + 0.5)
y = int(y + 0.5) y = int(y + 0.5)

4
input_reader.py
View file

@ -36,7 +36,7 @@ class EscapiReader(VideoReader):
self.fps = fps self.fps = fps
self.device = capture self.device = capture
escapi.count_capture_devices() escapi.count_capture_devices()
self.name = str(escapi.device_name(self.device).decode('utf8')) self.name = str(escapi.device_name(self.device).decode('utf8', 'surrogateescape'))
self.buffer = escapi.init_camera(self.device, self.width, self.height, self.fps) self.buffer = escapi.init_camera(self.device, self.width, self.height, self.fps)
escapi.do_capture(self.device) escapi.do_capture(self.device)
def is_open(self): def is_open(self):
@ -213,7 +213,7 @@ class InputReader():
devices = escapi.count_capture_devices() devices = escapi.count_capture_devices()
found = None found = None
for i in range(devices): for i in range(devices):
escapi_name = str(escapi.device_name(i).decode('utf8')) escapi_name = str(escapi.device_name(i).decode('utf8', 'surrogateescape'))
if name == escapi_name: if name == escapi_name:
found = i found = i
if found is None: if found is None:

15
tracker.py
View file

@ -140,7 +140,7 @@ def worker_thread(session, frame, input, crop_info, queue, input_name, idx, trac
conf, lms = tracker.landmarks(output[0], crop_info) conf, lms = tracker.landmarks(output[0], crop_info)
if conf > tracker.threshold: if conf > tracker.threshold:
try: try:
eye_state = tracker.get_eye_state(frame, lms, single=True) eye_state = tracker.get_eye_state(frame, lms)
except: except:
eye_state = [(1.0, 0.0, 0.0, 0.0), (1.0, 0.0, 0.0, 0.0)] eye_state = [(1.0, 0.0, 0.0, 0.0), (1.0, 0.0, 0.0, 0.0)]
queue.put((session, conf, (lms, eye_state), crop_info, idx)) queue.put((session, conf, (lms, eye_state), crop_info, idx))
@ -548,14 +548,12 @@ class Tracker():
self.input_name = self.session.get_inputs()[0].name self.input_name = self.session.get_inputs()[0].name
options = onnxruntime.SessionOptions() options = onnxruntime.SessionOptions()
options.inter_op_num_threads = 1 #options.intra_op_num_threads = max(max_threads,4)
options.intra_op_num_threads = max(max_threads,4) options.intra_op_num_threads = 1
options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
options.log_severity_level = 3 options.log_severity_level = 3
self.gaze_model = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options) self.gaze_model = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options)
options.intra_op_num_threads = 1
self.gaze_model_single = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options)
self.detection = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_detection_opt.onnx"), sess_options=options) self.detection = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_detection_opt.onnx"), sess_options=options)
self.faces = [] self.faces = []
@ -922,7 +920,7 @@ class Tracker():
frame = frame[y1:y2, x1:x2] frame = frame[y1:y2, x1:x2]
return frame, lms, offset return frame, lms, offset
def get_eye_state(self, frame, lms, single=False): def get_eye_state(self, frame, lms):
if self.no_gaze: if self.no_gaze:
return [(1.0, 0.0, 0.0, 0.0), (1.0, 0.0, 0.0, 0.0)] return [(1.0, 0.0, 0.0, 0.0), (1.0, 0.0, 0.0, 0.0)]
lms = np.array(lms) lms = np.array(lms)
@ -938,10 +936,7 @@ class Tracker():
return [(1.0, 0.0, 0.0, 0.0), (1.0, 0.0, 0.0, 0.0)] return [(1.0, 0.0, 0.0, 0.0), (1.0, 0.0, 0.0, 0.0)]
both_eyes = np.concatenate((right_eye, left_eye)) both_eyes = np.concatenate((right_eye, left_eye))
results = None results = None
if single: results = self.gaze_model.run([], {self.input_name: both_eyes})
results = self.gaze_model_single.run([], {self.input_name: both_eyes})
else:
results = self.gaze_model.run([], {self.input_name: both_eyes})
open = [0, 0] open = [0, 0]
open[0] = 1#results[1][0].argmax() open[0] = 1#results[1][0].argmax()
open[1] = 1#results[1][1].argmax() open[1] = 1#results[1][1].argmax()