From 22384e341a73567232830bf5668cda2f9975d43d Mon Sep 17 00:00:00 2001 From: Emiliana <38952746+emilianavt@users.noreply.github.com> Date: Sat, 7 Aug 2021 15:55:29 +0200 Subject: [PATCH 1/6] Fixed data logging. --- facetracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/facetracker.py b/facetracker.py index f7d4dcc..96c4ad3 100644 --- a/facetracker.py +++ b/facetracker.py @@ -293,7 +293,7 @@ try: packet.extend(bytearray(struct.pack("f", f.translation[1]))) packet.extend(bytearray(struct.pack("f", f.translation[2]))) if not log is None: - log.write(f"{frame_count},{now},{width},{height},{args.fps},{face_num},{f.id},{f.eye_blink[0]},{f.eye_blink[1]},{f.conf},{f.success},{f.pnp_error},{f.quaternion[0]},{f.quaternion[1]},{f.quaternion[2]},{f.quaternion[3]},{f.euler[0]},{f.euler[1]},{f.euler[2]},{f.rotation[0]},{f.rotation[1]},{f.rotation[2]},{f.translation[0]},{f.translation[1]},{f.translation[2]}") + log.write(f"{frame_count},{now},{width},{height},{fps},{face_num},{f.id},{f.eye_blink[0]},{f.eye_blink[1]},{f.conf},{f.success},{f.pnp_error},{f.quaternion[0]},{f.quaternion[1]},{f.quaternion[2]},{f.quaternion[3]},{f.euler[0]},{f.euler[1]},{f.euler[2]},{f.rotation[0]},{f.rotation[1]},{f.rotation[2]},{f.translation[0]},{f.translation[1]},{f.translation[2]}") for (x,y,c) in f.lms: packet.extend(bytearray(struct.pack("f", c))) if args.visualize > 1: From 84c81f6938b0cc8b4608c0b4cfb37de4e374685a Mon Sep 17 00:00:00 2001 From: Emiliana <38952746+emilianavt@users.noreply.github.com> Date: Sat, 7 Aug 2021 17:47:14 +0200 Subject: [PATCH 2/6] Changed utf8 decoding to avoid crashes with invalid characters. --- dshowcapture.py | 6 +++--- input_reader.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dshowcapture.py b/dshowcapture.py index 6ac5f96..46def39 100644 --- a/dshowcapture.py +++ b/dshowcapture.py @@ -104,7 +104,7 @@ class DShowCapture(): def get_device(self, device_number): self.lib.get_device(self.cap, device_number, self.name_buffer, 255) - name_str = str(self.name_buffer.value.decode('utf8')) + name_str = str(self.name_buffer.value.decode('utf8', 'surrogateescape')) return name_str def get_info(self): @@ -113,7 +113,7 @@ class DShowCapture(): json_length = self.lib.get_json_length(self.cap); json_buffer = create_string_buffer(json_length) self.lib.get_json(self.cap, json_buffer, json_length); - json_text = str(json_buffer.value.decode('utf8')) + json_text = str(json_buffer.value.decode('utf8', 'surrogateescape')) self.info = json.loads(json_text) for cam in self.info: cam["type"] = "DirectShow" @@ -124,7 +124,7 @@ class DShowCapture(): json_length = self.bm_lib.get_json_length(); json_buffer = create_string_buffer(json_length) self.bm_lib.get_json(json_buffer, json_length); - json_text = str(json_buffer.value.decode('utf8')) + json_text = str(json_buffer.value.decode('utf8', 'surrogateescape')) bm_info = json.loads(json_text) dshow_len = len(self.info) for cam in bm_info: diff --git a/input_reader.py b/input_reader.py index 30e7bb6..a62af69 100644 --- a/input_reader.py +++ b/input_reader.py @@ -36,7 +36,7 @@ class EscapiReader(VideoReader): self.fps = fps self.device = capture escapi.count_capture_devices() - self.name = str(escapi.device_name(self.device).decode('utf8')) + self.name = str(escapi.device_name(self.device).decode('utf8', 'surrogateescape')) self.buffer = escapi.init_camera(self.device, self.width, self.height, self.fps) escapi.do_capture(self.device) def is_open(self): @@ -212,7 +212,7 @@ class InputReader(): devices = escapi.count_capture_devices() found = None for i in range(devices): - escapi_name = str(escapi.device_name(i).decode('utf8')) + escapi_name = str(escapi.device_name(i).decode('utf8', 'surrogateescape')) if name == escapi_name: found = i if found is None: From 24f26d049f64753d427246517e62938c8f329e66 Mon Sep 17 00:00:00 2001 From: Emiliana <38952746+emilianavt@users.noreply.github.com> Date: Thu, 12 Aug 2021 13:51:44 +0200 Subject: [PATCH 3/6] Updated README. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 58f2457..3ec75c3 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # Overview -**Note**: This is a tracking library, **not** a stand-alone avatar puppeteering program. I'm also working on [VSeeFace](https://www.vseeface.icu/), which allows animating [VRM](https://vrm.dev/en/how_to_make_vrm/) and [VSFAvatar](https://www.youtube.com/watch?v=jhQ8DF87I5I) 3D models by using OpenSeeFace tracking. [VTube Studio](https://denchisoft.com/) uses OpenSeeFace for webcam based tracking to animate Live2D models. +**Note**: This is a tracking library, **not** a stand-alone avatar puppeteering program. I'm also working on [VSeeFace](https://www.vseeface.icu/), which allows animating [VRM](https://vrm.dev/en/how_to_make_vrm/) and [VSFAvatar](https://www.youtube.com/watch?v=jhQ8DF87I5I) 3D models by using OpenSeeFace tracking. [VTube Studio](https://denchisoft.com/) uses OpenSeeFace for webcam based tracking to animate Live2D models. A fork working with the Godot engine can be found [here](https://github.com/you-win/openseeface-gd). This project implements a facial landmark detection model based on MobileNetV3. From c5ff6a8fe1c4bb11223d7bdbd53b395a6501395f Mon Sep 17 00:00:00 2001 From: Emiliana <38952746+emilianavt@users.noreply.github.com> Date: Thu, 12 Aug 2021 13:52:45 +0200 Subject: [PATCH 4/6] Updated README. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3ec75c3..2db1fed 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # Overview -**Note**: This is a tracking library, **not** a stand-alone avatar puppeteering program. I'm also working on [VSeeFace](https://www.vseeface.icu/), which allows animating [VRM](https://vrm.dev/en/how_to_make_vrm/) and [VSFAvatar](https://www.youtube.com/watch?v=jhQ8DF87I5I) 3D models by using OpenSeeFace tracking. [VTube Studio](https://denchisoft.com/) uses OpenSeeFace for webcam based tracking to animate Live2D models. A fork working with the Godot engine can be found [here](https://github.com/you-win/openseeface-gd). +**Note**: This is a tracking library, **not** a stand-alone avatar puppeteering program. I'm also working on [VSeeFace](https://www.vseeface.icu/), which allows animating [VRM](https://vrm.dev/en/how_to_make_vrm/) and [VSFAvatar](https://www.youtube.com/watch?v=jhQ8DF87I5I) 3D models by using OpenSeeFace tracking. [VTube Studio](https://denchisoft.com/) uses OpenSeeFace for webcam based tracking to animate Live2D models. A renderer for the Godot engine can be found [here](https://github.com/you-win/openseeface-gd). This project implements a facial landmark detection model based on MobileNetV3. From 44d97a967c701ecedc34b52532cd6e5eb2e82a07 Mon Sep 17 00:00:00 2001 From: Emiliana <38952746+emilianavt@users.noreply.github.com> Date: Sat, 11 Sep 2021 15:56:25 +0200 Subject: [PATCH 5/6] Adjusted eye landmark display threshold. --- facetracker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/facetracker.py b/facetracker.py index 96c4ad3..803359b 100644 --- a/facetracker.py +++ b/facetracker.py @@ -305,9 +305,9 @@ try: packet.extend(bytearray(struct.pack("f", x))) if not log is None: log.write(f",{y},{x},{c}") - if pt_num == 66 and (f.eye_blink[0] < 0.30 or c < 0.30): + if pt_num == 66 and (f.eye_blink[0] < 0.30 or c < 0.20): continue - if pt_num == 67 and (f.eye_blink[1] < 0.30 or c < 0.30): + if pt_num == 67 and (f.eye_blink[1] < 0.30 or c < 0.20): continue x = int(x + 0.5) y = int(y + 0.5) From 1264cc255cb233627a62ed83607361f8132dfde2 Mon Sep 17 00:00:00 2001 From: Emiliana <38952746+emilianavt@users.noreply.github.com> Date: Sat, 11 Sep 2021 15:57:03 +0200 Subject: [PATCH 6/6] Changed gaze tracking to always use single thread to avoid excessive CPU usage. --- tracker.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/tracker.py b/tracker.py index a22190b..f5cbcc7 100644 --- a/tracker.py +++ b/tracker.py @@ -140,7 +140,7 @@ def worker_thread(session, frame, input, crop_info, queue, input_name, idx, trac conf, lms = tracker.landmarks(output[0], crop_info) if conf > tracker.threshold: try: - eye_state = tracker.get_eye_state(frame, lms, single=True) + eye_state = tracker.get_eye_state(frame, lms) except: eye_state = [(1.0, 0.0, 0.0, 0.0), (1.0, 0.0, 0.0, 0.0)] queue.put((session, conf, (lms, eye_state), crop_info, idx)) @@ -548,14 +548,12 @@ class Tracker(): self.input_name = self.session.get_inputs()[0].name options = onnxruntime.SessionOptions() - options.inter_op_num_threads = 1 - options.intra_op_num_threads = max(max_threads,4) + #options.intra_op_num_threads = max(max_threads,4) + options.intra_op_num_threads = 1 options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL options.log_severity_level = 3 self.gaze_model = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options) - options.intra_op_num_threads = 1 - self.gaze_model_single = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options) self.detection = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_detection_opt.onnx"), sess_options=options) self.faces = [] @@ -922,7 +920,7 @@ class Tracker(): frame = frame[y1:y2, x1:x2] return frame, lms, offset - def get_eye_state(self, frame, lms, single=False): + def get_eye_state(self, frame, lms): if self.no_gaze: return [(1.0, 0.0, 0.0, 0.0), (1.0, 0.0, 0.0, 0.0)] lms = np.array(lms) @@ -938,10 +936,7 @@ class Tracker(): return [(1.0, 0.0, 0.0, 0.0), (1.0, 0.0, 0.0, 0.0)] both_eyes = np.concatenate((right_eye, left_eye)) results = None - if single: - results = self.gaze_model_single.run([], {self.input_name: both_eyes}) - else: - results = self.gaze_model.run([], {self.input_name: both_eyes}) + results = self.gaze_model.run([], {self.input_name: both_eyes}) open = [0, 0] open[0] = 1#results[1][0].argmax() open[1] = 1#results[1][1].argmax()