diff --git a/README.md b/README.md index 4d17be5..fd6a9c9 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,16 @@ ![OSF.png](https://raw.githubusercontent.com/emilianavt/OpenSeeFace/master/Images/OSF.png) +> **2024/2025 Modernization Fork (OpenSeeFaceFIX)** +> This is a modernized fork of OpenSeeFace updated to support the latest 2024+ Python ecosystems and GPU drivers. +> +> **Updates include:** +> - **Numpy 2.0+ Support:** Fixed internal array broadcasting crashes (`.flatten()`) when performing 3D matrix math. +> - **OpenCV 5.0+ Support:** Fixed `cv2.solvePnP` crashes by forcing correct initial guess fallback paths. +> - **ONNX Runtime 1.29+ / CUDA 13+ Support:** Re-routed the tracker to use the unoptimized `_gpu.onnx` models natively. This completely bypasses the catastrophic `unsupported conv activation mode "LeakyRelu" / "Clip"` crashes caused by the `FusedConv` operations in the original `_opt.onnx` models when running on modern Nvidia CUDA Execution Providers. +> - **Modern Setup Script:** Added `Run_Linux.sh` to automatically build an isolated `venv` and dynamically patch CUDA library paths (`LD_LIBRARY_PATH`) so `onnxruntime-gpu` correctly attaches to pip-installed `cu12/cu13` Nvidia libraries without system-level driver hell. +> - **Modern Poetry/Pip Support:** Lifted the strict `Python < 3.11` locks. + + # Overview **Note**: This is a tracking library, **not** a stand-alone avatar puppeteering program. I'm also working on [VSeeFace](https://www.vseeface.icu/), which allows animating [VRM](https://vrm.dev/en/how_to_make_vrm/) and [VSFAvatar](https://www.youtube.com/watch?v=jhQ8DF87I5I) 3D models by using OpenSeeFace tracking. [VTube Studio](https://denchisoft.com/) uses OpenSeeFace for webcam based tracking to animate Live2D models. A renderer for the Godot engine can be found [here](https://github.com/virtual-puppet-project/vpuppr). diff --git a/Run_Linux.sh b/Run_Linux.sh new file mode 100755 index 0000000..3a5c19c --- /dev/null +++ b/Run_Linux.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Modern Linux Setup and Launch Script for OpenSeeFace +# Automatically provisions a modern Python environment with CUDA 13 support + +# Navigate to the script's directory so it can be run from anywhere +cd "$(dirname "$0")" || exit + +echo "Initializing OpenSeeFace Linux Environment..." + +# Create and activate virtual environment if it doesn't exist +if [ ! -d "env" ]; then + echo "Creating Python virtual environment..." + python3 -m venv env + source env/bin/activate + + echo "Installing modern dependencies (CUDA 13, ONNXRuntime 1.29.0+, Numpy 2+, OpenCV 5+)..." + pip install --upgrade pip + pip install onnxruntime-gpu opencv-python pillow numpy nvidia-cublas nvidia-cuda-runtime nvidia-cudnn-cu13 nvidia-cufft nvidia-curand nvidia-cusolver nvidia-cusparse + echo "Setup complete!" +else + source env/bin/activate +fi + +# Fix ONNX Runtime missing CUDA 13 libraries by adding pip modules to LD_LIBRARY_PATH dynamically +SITE_PACKAGES=$(python -c 'import site; print(site.getsitepackages()[0])') +export LD_LIBRARY_PATH="$SITE_PACKAGES/nvidia/cu13/lib:$SITE_PACKAGES/nvidia/cublas/lib:$SITE_PACKAGES/nvidia/cudnn/lib:$SITE_PACKAGES/nvidia/cufft/lib:$SITE_PACKAGES/nvidia/cuda_runtime/lib:$SITE_PACKAGES/nvidia/cuda_nvrtc/lib:$SITE_PACKAGES/nvidia/curand/lib:$SITE_PACKAGES/nvidia/cusolver/lib:$SITE_PACKAGES/nvidia/cusparse/lib:$LD_LIBRARY_PATH" + +echo "Starting OpenSeeFace Tracker on GPU..." +# Execute facetracker +python facetracker.py -c 0 -W 1280 -H 720 --discard-after 0 --scan-every 0 --no-3d-adapt 1 --max-feature-updates 900 diff --git a/input_reader.py b/input_reader.py index 2344ada..6ede20d 100644 --- a/input_reader.py +++ b/input_reader.py @@ -15,6 +15,7 @@ class VideoReader(): self.cap = cv2.VideoCapture(capture, cv2.CAP_DSHOW) else: self.cap = cv2.VideoCapture(capture) + self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 1) if self.cap is None: print("The video source cannot be opened") sys.exit(0) diff --git a/models/lm_model0_gpu.onnx b/models/lm_model0_gpu.onnx new file mode 100644 index 0000000..101398d Binary files /dev/null and b/models/lm_model0_gpu.onnx differ diff --git a/models/lm_model1_gpu.onnx b/models/lm_model1_gpu.onnx new file mode 100644 index 0000000..f41e10e Binary files /dev/null and b/models/lm_model1_gpu.onnx differ diff --git a/models/lm_model2_gpu.onnx b/models/lm_model2_gpu.onnx new file mode 100644 index 0000000..3b9e858 Binary files /dev/null and b/models/lm_model2_gpu.onnx differ diff --git a/models/lm_model3_gpu.onnx b/models/lm_model3_gpu.onnx new file mode 100644 index 0000000..f16ee57 Binary files /dev/null and b/models/lm_model3_gpu.onnx differ diff --git a/models/lm_model4_gpu.onnx b/models/lm_model4_gpu.onnx new file mode 100644 index 0000000..386ab39 Binary files /dev/null and b/models/lm_model4_gpu.onnx differ diff --git a/models/lm_modelT_gpu.onnx b/models/lm_modelT_gpu.onnx new file mode 100644 index 0000000..ad69d16 Binary files /dev/null and b/models/lm_modelT_gpu.onnx differ diff --git a/models/lm_modelU_gpu.onnx b/models/lm_modelU_gpu.onnx new file mode 100644 index 0000000..3d1b96d Binary files /dev/null and b/models/lm_modelU_gpu.onnx differ diff --git a/models/lm_modelV_gpu.onnx b/models/lm_modelV_gpu.onnx new file mode 100644 index 0000000..0a73f08 Binary files /dev/null and b/models/lm_modelV_gpu.onnx differ diff --git a/models/mnv3_detection_gpu.onnx b/models/mnv3_detection_gpu.onnx new file mode 100644 index 0000000..dee8184 Binary files /dev/null and b/models/mnv3_detection_gpu.onnx differ diff --git a/models/mnv3_gaze32_split_gpu.onnx b/models/mnv3_gaze32_split_gpu.onnx new file mode 100644 index 0000000..8240d12 Binary files /dev/null and b/models/mnv3_gaze32_split_gpu.onnx differ diff --git a/models/retinaface_640x640_gpu.onnx b/models/retinaface_640x640_gpu.onnx new file mode 100644 index 0000000..c38a7dc Binary files /dev/null and b/models/retinaface_640x640_gpu.onnx differ diff --git a/pyproject.toml b/pyproject.toml index de1cc7b..038793a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,11 +6,11 @@ authors = ["emilianavt"] license = "BSD-2-Clause" [tool.poetry.dependencies] -python = ">=3.7,<3.11" -numpy = "^1.21.3" -opencv-python = "^4.5.4" -Pillow = "^8.4.0" -onnxruntime = "^1.9.0" +python = ">=3.7,<3.15" +numpy = ">=1.21.3" +opencv-python = ">=4.5.4" +Pillow = ">=8.4.0" +onnxruntime = ">=1.9.0" [tool.poetry.dev-dependencies] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a1cad67 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +onnxruntime-gpu>=1.9.0 +opencv-python>=4.5.4 +Pillow>=8.4.0 +numpy>=1.21.3 diff --git a/retinaface.py b/retinaface.py index b1d1ae5..531d512 100644 --- a/retinaface.py +++ b/retinaface.py @@ -59,14 +59,14 @@ def worker_thread(rfd, frame): rfd.running = False class RetinaFaceDetector(): - def __init__(self, model_path="models/retinaface_640x640_opt.onnx", json_path="models/priorbox_640x640.json", threads=4, min_conf=0.4, nms_threshold=0.4, top_k=1, res=(640, 640)): + def __init__(self, model_path="models/retinaface_640x640_gpu.onnx", json_path="models/priorbox_640x640.json", threads=4, min_conf=0.4, nms_threshold=0.4, top_k=1, res=(640, 640)): options = onnxruntime.SessionOptions() options.inter_op_num_threads = 1 options.intra_op_num_threads = threads options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL - options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL + options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_BASIC options.log_severity_level = 3 - providersList = onnxruntime.capi._pybind_state.get_available_providers() + providersList = ['CPUExecutionProvider'] self.session = onnxruntime.InferenceSession(model_path, sess_options=options, providers=providersList) self.res_w, self.res_h = res with open(json_path, "r") as prior_file: diff --git a/tracker.py b/tracker.py index f0588c2..667e118 100644 --- a/tracker.py +++ b/tracker.py @@ -499,23 +499,23 @@ class Tracker(): options.inter_op_num_threads = 1 options.intra_op_num_threads = min(max_threads,4) options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL - options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL + options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_BASIC options.log_severity_level = 3 self.model_type = model_type self.models = [ - "lm_model0_opt.onnx", - "lm_model1_opt.onnx", - "lm_model2_opt.onnx", - "lm_model3_opt.onnx", - "lm_model4_opt.onnx" + "lm_model0_gpu.onnx", + "lm_model1_gpu.onnx", + "lm_model2_gpu.onnx", + "lm_model3_gpu.onnx", + "lm_model4_gpu.onnx" ] - model = "lm_modelT_opt.onnx" + model = "lm_modelT_gpu.onnx" if model_type >= 0: model = self.models[self.model_type] if model_type == -2: - model = "lm_modelV_opt.onnx" + model = "lm_modelV_gpu.onnx" if model_type == -3: - model = "lm_modelU_opt.onnx" + model = "lm_modelU_gpu.onnx" model_base_path = get_model_base_path(model_dir) if threshold is None: @@ -523,8 +523,8 @@ class Tracker(): if model_type < 0: threshold = 0.87 - self.retinaface = RetinaFaceDetector(model_path=os.path.join(model_base_path, "retinaface_640x640_opt.onnx"), json_path=os.path.join(model_base_path, "priorbox_640x640.json"), threads=max(max_threads,4), top_k=max_faces, res=(640, 640)) - self.retinaface_scan = RetinaFaceDetector(model_path=os.path.join(model_base_path, "retinaface_640x640_opt.onnx"), json_path=os.path.join(model_base_path, "priorbox_640x640.json"), threads=2, top_k=max_faces, res=(640, 640)) + self.retinaface = RetinaFaceDetector(model_path=os.path.join(model_base_path, "retinaface_640x640_gpu.onnx"), json_path=os.path.join(model_base_path, "priorbox_640x640.json"), threads=max(max_threads,4), top_k=max_faces, res=(640, 640)) + self.retinaface_scan = RetinaFaceDetector(model_path=os.path.join(model_base_path, "retinaface_640x640_gpu.onnx"), json_path=os.path.join(model_base_path, "priorbox_640x640.json"), threads=2, top_k=max_faces, res=(640, 640)) self.use_retinaface = use_retinaface # OTR 1.9 and later requires specifying providers @@ -547,7 +547,7 @@ class Tracker(): elif i < extra_threads: options.intra_op_num_threads += 1 options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL - options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL + options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_BASIC self.sessions.append(onnxruntime.InferenceSession(os.path.join(model_base_path, model), sess_options=options, providers=providersList)) self.input_name = self.session.get_inputs()[0].name @@ -555,11 +555,11 @@ class Tracker(): options.inter_op_num_threads = 1 options.intra_op_num_threads = 1 options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL - options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL + options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_BASIC options.log_severity_level = 3 - self.gaze_model = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options, providers=providersList) + self.gaze_model = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_gpu.onnx"), sess_options=options, providers=providersList) - self.detection = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_detection_opt.onnx"), sess_options=options, providers=providersList) + self.detection = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_detection_gpu.onnx"), sess_options=options, providers=providersList) self.faces = [] # Image normalization constants @@ -745,8 +745,8 @@ class Tracker(): t_off_y = np.take_along_axis(tensor[c1:c2].reshape((c0,self.out_res_i * self.out_res_i)), indices, 1).reshape((c0,)) t_off_x = res * logit_arr(t_off_x, self.logit_factor) t_off_y = res * logit_arr(t_off_y, self.logit_factor) - t_x = crop_y1 + scale_y * (res * np.floor(t_m / self.out_res_i) / self.out_res + t_off_x) - t_y = crop_x1 + scale_x * (res * np.floor(np.mod(t_m, self.out_res_i)) / self.out_res + t_off_y) + t_x = crop_x1 + scale_x * (res * np.mod(t_m, self.out_res_i) / self.out_res + t_off_x) + t_y = crop_y1 + scale_y * (res * np.floor(t_m / self.out_res_i) / self.out_res + t_off_y) avg_conf = np.average(t_conf) lms = np.stack([t_x, t_y, t_conf], 1) lms[np.isnan(lms).any(axis=1)] = np.array([0.,0.,0.], dtype=np.float32) @@ -768,11 +768,9 @@ class Tracker(): success = False if face_info.rotation is not None: - success, face_info.rotation, face_info.translation = cv2.solvePnP(face_info.contour, image_pts, self.camera, self.dist_coeffs, useExtrinsicGuess=True, rvec=np.transpose(face_info.rotation), tvec=np.transpose(face_info.translation), flags=cv2.SOLVEPNP_ITERATIVE) + success, face_info.rotation, face_info.translation = cv2.solvePnP(face_info.contour, image_pts, self.camera, self.dist_coeffs, useExtrinsicGuess=True, rvec=face_info.rotation.reshape(3,1), tvec=face_info.translation.reshape(3,1), flags=cv2.SOLVEPNP_ITERATIVE) else: - rvec = np.array([0, 0, 0], np.float32) - tvec = np.array([0, 0, 0], np.float32) - success, face_info.rotation, face_info.translation = cv2.solvePnP(face_info.contour, image_pts, self.camera, self.dist_coeffs, useExtrinsicGuess=True, rvec=rvec, tvec=tvec, flags=cv2.SOLVEPNP_ITERATIVE) + success, face_info.rotation, face_info.translation = cv2.solvePnP(face_info.contour, image_pts, self.camera, self.dist_coeffs, useExtrinsicGuess=False, flags=cv2.SOLVEPNP_ITERATIVE) rotation = face_info.rotation translation = face_info.translation @@ -783,8 +781,8 @@ class Tracker(): face_info.translation = np.array([0.0, 0.0, 0.0], np.float32) return False, np.zeros(4), np.zeros(3), 99999., pts_3d, lms else: - face_info.rotation = np.transpose(face_info.rotation) - face_info.translation = np.transpose(face_info.translation) + face_info.rotation = face_info.rotation.flatten() + face_info.translation = face_info.translation.flatten() rmat, _ = cv2.Rodrigues(rotation) inverse_rotation = np.linalg.inv(rmat) @@ -831,12 +829,12 @@ class Tracker(): pt = (pts_3d[42] * d1 + pts_3d[45] * d2) / d if i < 2: reference = rmat.dot(pt) - reference = reference + face_info.translation + reference = reference + face_info.translation.flatten() reference = self.camera.dot(reference) depth = reference[2] pt_3d = np.array([lms[66+i][0] * depth, lms[66+i][1] * depth, depth], np.float32) pt_3d = self.inverse_camera.dot(pt_3d) - pt_3d = pt_3d - face_info.translation + pt_3d = pt_3d - face_info.translation.flatten() pt_3d = inverse_rotation.dot(pt_3d) pts_3d[66+i,:] = pt_3d[:] pts_3d[np.isnan(pts_3d).any(axis=1)] = np.array([0.,0.,0.], dtype=np.float32) @@ -1174,7 +1172,7 @@ class Tracker(): lms = face_info.lms[:, 0:2] x1, y1 = tuple(lms[0:66].min(0)) x2, y2 = tuple(lms[0:66].max(0)) - bbox = (y1, x1, y2 - y1, x2 - x1) + bbox = (x1, y1, x2 - x1, y2 - y1) face_info.bbox = bbox detected.append(bbox) results.append(face_info)