Merge 053ea8c612 into 85aa70fc67

README.md

@@ -1,5 +1,16 @@
 ![OSF.png](https://raw.githubusercontent.com/emilianavt/OpenSeeFace/master/Images/OSF.png)
 
+> **2024/2025 Modernization Fork (OpenSeeFaceFIX)**
+> This is a modernized fork of OpenSeeFace updated to support the latest 2024+ Python ecosystems and GPU drivers.
+> 
+> **Updates include:**
+> - **Numpy 2.0+ Support:** Fixed internal array broadcasting crashes (`.flatten()`) when performing 3D matrix math.
+> - **OpenCV 5.0+ Support:** Fixed `cv2.solvePnP` crashes by forcing correct initial guess fallback paths.
+> - **ONNX Runtime 1.29+ / CUDA 13+ Support:** Re-routed the tracker to use the unoptimized `_gpu.onnx` models natively. This completely bypasses the catastrophic `unsupported conv activation mode "LeakyRelu" / "Clip"` crashes caused by the `FusedConv` operations in the original `_opt.onnx` models when running on modern Nvidia CUDA Execution Providers.
+> - **Modern Setup Script:** Added `Run_Linux.sh` to automatically build an isolated `venv` and dynamically patch CUDA library paths (`LD_LIBRARY_PATH`) so `onnxruntime-gpu` correctly attaches to pip-installed `cu12/cu13` Nvidia libraries without system-level driver hell.
+> - **Modern Poetry/Pip Support:** Lifted the strict `Python < 3.11` locks.
+
+
 # Overview
 
 **Note**: This is a tracking library, **not** a stand-alone avatar puppeteering program. I'm also working on [VSeeFace](https://www.vseeface.icu/), which allows animating [VRM](https://vrm.dev/en/how_to_make_vrm/) and [VSFAvatar](https://www.youtube.com/watch?v=jhQ8DF87I5I) 3D models by using OpenSeeFace tracking. [VTube Studio](https://denchisoft.com/) uses OpenSeeFace for webcam based tracking to animate Live2D models. A renderer for the Godot engine can be found [here](https://github.com/virtual-puppet-project/vpuppr).

Run_Linux.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# Modern Linux Setup and Launch Script for OpenSeeFace
+# Automatically provisions a modern Python environment with CUDA 13 support
+
+# Navigate to the script's directory so it can be run from anywhere
+cd "$(dirname "$0")" || exit
+
+echo "Initializing OpenSeeFace Linux Environment..."
+
+# Create and activate virtual environment if it doesn't exist
+if [ ! -d "env" ]; then
+    echo "Creating Python virtual environment..."
+    python3 -m venv env
+    source env/bin/activate
+    
+    echo "Installing modern dependencies (CUDA 13, ONNXRuntime 1.29.0+, Numpy 2+, OpenCV 5+)..."
+    pip install --upgrade pip
+    pip install onnxruntime-gpu opencv-python pillow numpy nvidia-cublas nvidia-cuda-runtime nvidia-cudnn-cu13 nvidia-cufft nvidia-curand nvidia-cusolver nvidia-cusparse
+    echo "Setup complete!"
+else
+    source env/bin/activate
+fi
+
+# Fix ONNX Runtime missing CUDA 13 libraries by adding pip modules to LD_LIBRARY_PATH dynamically
+SITE_PACKAGES=$(python -c 'import site; print(site.getsitepackages()[0])')
+export LD_LIBRARY_PATH="$SITE_PACKAGES/nvidia/cu13/lib:$SITE_PACKAGES/nvidia/cublas/lib:$SITE_PACKAGES/nvidia/cudnn/lib:$SITE_PACKAGES/nvidia/cufft/lib:$SITE_PACKAGES/nvidia/cuda_runtime/lib:$SITE_PACKAGES/nvidia/cuda_nvrtc/lib:$SITE_PACKAGES/nvidia/curand/lib:$SITE_PACKAGES/nvidia/cusolver/lib:$SITE_PACKAGES/nvidia/cusparse/lib:$LD_LIBRARY_PATH"
+
+echo "Starting OpenSeeFace Tracker on GPU..."
+# Execute facetracker
+python facetracker.py -c 0 -W 1280 -H 720 --discard-after 0 --scan-every 0 --no-3d-adapt 1 --max-feature-updates 900

input_reader.py

@@ -15,6 +15,7 @@ class VideoReader():
             self.cap = cv2.VideoCapture(capture, cv2.CAP_DSHOW)
         else:
             self.cap = cv2.VideoCapture(capture)
+            self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
         if self.cap is None:
             print("The video source cannot be opened")
             sys.exit(0)

pyproject.toml

@@ -6,11 +6,11 @@ authors = ["emilianavt"]
 license = "BSD-2-Clause"
 
 [tool.poetry.dependencies]
-python = ">=3.7,<3.11"
+python = ">=3.7,<3.15"
-numpy = "^1.21.3"
+numpy = ">=1.21.3"
-opencv-python = "^4.5.4"
+opencv-python = ">=4.5.4"
-Pillow = "^8.4.0"
+Pillow = ">=8.4.0"
-onnxruntime = "^1.9.0"
+onnxruntime = ">=1.9.0"
 
 [tool.poetry.dev-dependencies]
 

requirements.txt

@@ -0,0 +1,4 @@
+onnxruntime-gpu>=1.9.0
+opencv-python>=4.5.4
+Pillow>=8.4.0
+numpy>=1.21.3

retinaface.py

@@ -59,14 +59,14 @@ def worker_thread(rfd, frame):
     rfd.running = False
 
 class RetinaFaceDetector():
-    def __init__(self, model_path="models/retinaface_640x640_opt.onnx", json_path="models/priorbox_640x640.json", threads=4, min_conf=0.4, nms_threshold=0.4, top_k=1, res=(640, 640)):
+    def __init__(self, model_path="models/retinaface_640x640_gpu.onnx", json_path="models/priorbox_640x640.json", threads=4, min_conf=0.4, nms_threshold=0.4, top_k=1, res=(640, 640)):
         options = onnxruntime.SessionOptions()
         options.inter_op_num_threads = 1
         options.intra_op_num_threads = threads
         options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
-        options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
+        options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_BASIC
         options.log_severity_level = 3
-        providersList = onnxruntime.capi._pybind_state.get_available_providers()
+        providersList = ['CPUExecutionProvider']
         self.session = onnxruntime.InferenceSession(model_path, sess_options=options, providers=providersList)
         self.res_w, self.res_h = res
         with open(json_path, "r") as prior_file:

tracker.py

@@ -499,23 +499,23 @@ class Tracker():
         options.inter_op_num_threads = 1
         options.intra_op_num_threads = min(max_threads,4)
         options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
-        options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
+        options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_BASIC
         options.log_severity_level = 3
         self.model_type = model_type
         self.models = [
-            "lm_model0_opt.onnx",
+            "lm_model0_gpu.onnx",
-            "lm_model1_opt.onnx",
+            "lm_model1_gpu.onnx",
-            "lm_model2_opt.onnx",
+            "lm_model2_gpu.onnx",
-            "lm_model3_opt.onnx",
+            "lm_model3_gpu.onnx",
-            "lm_model4_opt.onnx"
+            "lm_model4_gpu.onnx"
         ]
-        model = "lm_modelT_opt.onnx"
+        model = "lm_modelT_gpu.onnx"
         if model_type >= 0:
             model = self.models[self.model_type]
         if model_type == -2:
-            model = "lm_modelV_opt.onnx"
+            model = "lm_modelV_gpu.onnx"
         if model_type == -3:
-            model = "lm_modelU_opt.onnx"
+            model = "lm_modelU_gpu.onnx"
         model_base_path = get_model_base_path(model_dir)
 
         if threshold is None:
@@ -523,8 +523,8 @@ class Tracker():
             if model_type < 0:
                 threshold = 0.87
 
-        self.retinaface = RetinaFaceDetector(model_path=os.path.join(model_base_path, "retinaface_640x640_opt.onnx"), json_path=os.path.join(model_base_path, "priorbox_640x640.json"), threads=max(max_threads,4), top_k=max_faces, res=(640, 640))
+        self.retinaface = RetinaFaceDetector(model_path=os.path.join(model_base_path, "retinaface_640x640_gpu.onnx"), json_path=os.path.join(model_base_path, "priorbox_640x640.json"), threads=max(max_threads,4), top_k=max_faces, res=(640, 640))
-        self.retinaface_scan = RetinaFaceDetector(model_path=os.path.join(model_base_path, "retinaface_640x640_opt.onnx"), json_path=os.path.join(model_base_path, "priorbox_640x640.json"), threads=2, top_k=max_faces, res=(640, 640))
+        self.retinaface_scan = RetinaFaceDetector(model_path=os.path.join(model_base_path, "retinaface_640x640_gpu.onnx"), json_path=os.path.join(model_base_path, "priorbox_640x640.json"), threads=2, top_k=max_faces, res=(640, 640))
         self.use_retinaface = use_retinaface
 
         # OTR 1.9 and later requires specifying providers
@@ -547,7 +547,7 @@ class Tracker():
             elif i < extra_threads:
                 options.intra_op_num_threads += 1
             options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
-            options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
+            options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_BASIC
             self.sessions.append(onnxruntime.InferenceSession(os.path.join(model_base_path, model), sess_options=options, providers=providersList))
         self.input_name = self.session.get_inputs()[0].name
 
@@ -555,11 +555,11 @@ class Tracker():
         options.inter_op_num_threads = 1
         options.intra_op_num_threads = 1
         options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
-        options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
+        options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_BASIC
         options.log_severity_level = 3
-        self.gaze_model = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options, providers=providersList)
+        self.gaze_model = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_gpu.onnx"), sess_options=options, providers=providersList)
 
-        self.detection = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_detection_opt.onnx"), sess_options=options, providers=providersList)
+        self.detection = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_detection_gpu.onnx"), sess_options=options, providers=providersList)
         self.faces = []
 
         # Image normalization constants
@@ -745,8 +745,8 @@ class Tracker():
         t_off_y = np.take_along_axis(tensor[c1:c2].reshape((c0,self.out_res_i * self.out_res_i)), indices, 1).reshape((c0,))
         t_off_x = res * logit_arr(t_off_x, self.logit_factor)
         t_off_y = res * logit_arr(t_off_y, self.logit_factor)
-        t_x = crop_y1 + scale_y * (res * np.floor(t_m / self.out_res_i) / self.out_res + t_off_x)
+        t_x = crop_x1 + scale_x * (res * np.mod(t_m, self.out_res_i) / self.out_res + t_off_x)
-        t_y = crop_x1 + scale_x * (res * np.floor(np.mod(t_m, self.out_res_i)) / self.out_res + t_off_y)
+        t_y = crop_y1 + scale_y * (res * np.floor(t_m / self.out_res_i) / self.out_res + t_off_y)
         avg_conf = np.average(t_conf)
         lms = np.stack([t_x, t_y, t_conf], 1)
         lms[np.isnan(lms).any(axis=1)] = np.array([0.,0.,0.], dtype=np.float32)
@@ -768,11 +768,9 @@ class Tracker():
 
         success = False
         if face_info.rotation is not None:
-            success, face_info.rotation, face_info.translation = cv2.solvePnP(face_info.contour, image_pts, self.camera, self.dist_coeffs, useExtrinsicGuess=True, rvec=np.transpose(face_info.rotation), tvec=np.transpose(face_info.translation), flags=cv2.SOLVEPNP_ITERATIVE)
+            success, face_info.rotation, face_info.translation = cv2.solvePnP(face_info.contour, image_pts, self.camera, self.dist_coeffs, useExtrinsicGuess=True, rvec=face_info.rotation.reshape(3,1), tvec=face_info.translation.reshape(3,1), flags=cv2.SOLVEPNP_ITERATIVE)
         else:
-            rvec = np.array([0, 0, 0], np.float32)
+            success, face_info.rotation, face_info.translation = cv2.solvePnP(face_info.contour, image_pts, self.camera, self.dist_coeffs, useExtrinsicGuess=False, flags=cv2.SOLVEPNP_ITERATIVE)
-            tvec = np.array([0, 0, 0], np.float32)
-            success, face_info.rotation, face_info.translation = cv2.solvePnP(face_info.contour, image_pts, self.camera, self.dist_coeffs, useExtrinsicGuess=True, rvec=rvec, tvec=tvec, flags=cv2.SOLVEPNP_ITERATIVE)
 
         rotation = face_info.rotation
         translation = face_info.translation
@@ -783,8 +781,8 @@ class Tracker():
             face_info.translation = np.array([0.0, 0.0, 0.0], np.float32)
             return False, np.zeros(4), np.zeros(3), 99999., pts_3d, lms
         else:
-            face_info.rotation = np.transpose(face_info.rotation)
+            face_info.rotation = face_info.rotation.flatten()
-            face_info.translation = np.transpose(face_info.translation)
+            face_info.translation = face_info.translation.flatten()
 
         rmat, _ = cv2.Rodrigues(rotation)
         inverse_rotation = np.linalg.inv(rmat)
@@ -831,12 +829,12 @@ class Tracker():
                 pt = (pts_3d[42] * d1 + pts_3d[45] * d2) / d
             if i < 2:
                 reference = rmat.dot(pt)
-                reference = reference + face_info.translation
+                reference = reference + face_info.translation.flatten()
                 reference = self.camera.dot(reference)
                 depth = reference[2]
                 pt_3d = np.array([lms[66+i][0] * depth, lms[66+i][1] * depth, depth], np.float32)
                 pt_3d = self.inverse_camera.dot(pt_3d)
-                pt_3d = pt_3d - face_info.translation
+                pt_3d = pt_3d - face_info.translation.flatten()
                 pt_3d = inverse_rotation.dot(pt_3d)
                 pts_3d[66+i,:] = pt_3d[:]
         pts_3d[np.isnan(pts_3d).any(axis=1)] = np.array([0.,0.,0.], dtype=np.float32)
@@ -1174,7 +1172,7 @@ class Tracker():
                 lms = face_info.lms[:, 0:2]
                 x1, y1 = tuple(lms[0:66].min(0))
                 x2, y2 = tuple(lms[0:66].max(0))
-                bbox = (y1, x1, y2 - y1, x2 - x1)
+                bbox = (x1, y1, x2 - x1, y2 - y1)
                 face_info.bbox = bbox
                 detected.append(bbox)
                 results.append(face_info)