diff --git a/.github/actions/upload-dated-artifact/action.yaml b/.github/actions/upload-dated-artifact/action.yaml new file mode 100644 index 0000000..7fc3fbe --- /dev/null +++ b/.github/actions/upload-dated-artifact/action.yaml @@ -0,0 +1,26 @@ +name: Upload dated artifact +description: Uploads an artifact with a datetime + +inputs: + name: + description: The name of the artifact + required: true + path: + description: The path to the file/folder to upload + required: true + +runs: + using: composite + steps: + # https://stackoverflow.com/questions/60942067/get-current-date-and-time-in-github-workflows + - name: Get current datetime + id: date + shell: bash + run: echo "::set-output name=date::$(date '+%Y-%m-%d_%H-%M-%S')" + + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.name }}_${{ steps.date.outputs.date }} + path: ${{ inputs.path }} + if-no-files-found: error diff --git a/.github/workflows/create-binaries.yaml b/.github/workflows/create-binaries.yaml new file mode 100644 index 0000000..26572bc --- /dev/null +++ b/.github/workflows/create-binaries.yaml @@ -0,0 +1,87 @@ +name: Package using PyInstaller +on: + push: + tags: + - v** + workflow_dispatch: + +jobs: + build-windows: + env: + dist-path: dist/OpenSeeFace + runs-on: windows-latest + steps: + - name: Pull OpenSeeFace + uses: actions/checkout@v3 + + - name: Create binary + shell: cmd + run: | + choco install vcredist140 + + xcopy c:\Windows\System32\msvcp140.dll . + xcopy c:\Windows\System32\vcomp140.dll . + xcopy c:\Windows\System32\concrt140.dll . + xcopy c:\Windows\System32\vccorlib140.dll . + + make_exe.bat + + - name: Move files for uploading + shell: bash + run: | + # Prevents the parent dist/ directory from being included + mv dist/facetracker ${{ env.dist-path }} + cp -r models ${{ env.dist-path }} + + - name: Upload artifact + uses: ./.github/actions/upload-dated-artifact + with: + name: windows + path: dist # Add the parent directory to prevent a tarbomb + + # Files must be packed into a tar in order for file permissions to be preserved + build-unix: + env: + artifact-path: OpenSeeFace + tar-path: OpenSeeFace.tar + strategy: + fail-fast: false + matrix: + include: + - name: linux + os: ubuntu-latest + - name: mac + os: macos-latest + runs-on: ${{ matrix.os }} + steps: + - name: Pull OpenSeeFace + uses: actions/checkout@v3 + + - name: Create binary + shell: bash + run: | + chmod +x make_exe.sh + ./make_exe.sh + + - name: Move files for uploading + shell: bash + run: | + # Prevents the parent dist/ directory from being included + mv dist/facetracker ${{ env.artifact-path }} + cp -r models ${{ env.artifact-path }} + + - name: Fix permissions from PyInstaller + shell: bash + run: | + find ${{ env.artifact-path }} -type f -exec chmod -x {} + + chmod +x ${{ env.artifact-path }}/facetracker + + - name: Pack files into tar + shell: bash + run: tar -cvf ${{ env.tar-path }} ${{ env.artifact-path }} + + - name: Upload artifact + uses: ./.github/actions/upload-dated-artifact + with: + name: ${{ matrix.name }} + path: ${{ env.tar-path }} diff --git a/README.md b/README.md index 58f2457..4d17be5 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # Overview -**Note**: This is a tracking library, **not** a stand-alone avatar puppeteering program. I'm also working on [VSeeFace](https://www.vseeface.icu/), which allows animating [VRM](https://vrm.dev/en/how_to_make_vrm/) and [VSFAvatar](https://www.youtube.com/watch?v=jhQ8DF87I5I) 3D models by using OpenSeeFace tracking. [VTube Studio](https://denchisoft.com/) uses OpenSeeFace for webcam based tracking to animate Live2D models. +**Note**: This is a tracking library, **not** a stand-alone avatar puppeteering program. I'm also working on [VSeeFace](https://www.vseeface.icu/), which allows animating [VRM](https://vrm.dev/en/how_to_make_vrm/) and [VSFAvatar](https://www.youtube.com/watch?v=jhQ8DF87I5I) 3D models by using OpenSeeFace tracking. [VTube Studio](https://denchisoft.com/) uses OpenSeeFace for webcam based tracking to animate Live2D models. A renderer for the Godot engine can be found [here](https://github.com/virtual-puppet-project/vpuppr). This project implements a facial landmark detection model based on MobileNetV3. @@ -12,6 +12,14 @@ If anyone is curious, the name is a silly pun on the open seas and seeing faces. An up to date sample video can be found [here](https://www.youtube.com/watch?v=AaNap_ud_3I&vq=hd1080), showing the default tracking model's performance under different noise and light levels. +# Tracking quality + +Since the landmarks used by OpenSeeFace are a bit different from those used by other approaches (they are close to iBUG 68, with two less points in the mouth corners and quasi-3D face contours instead of face contours that follow the visible outline) it is hard to numerically compare its accuracy to that of other approaches found commonly in scientific literature. The tracking performance is also more optimized for making landmarks that are useful for animating an avatar than for exactly fitting the face image. For example, as long as the eye landmarks show whether the eyes are opened or closed, even if their location is somewhat off, they can still be useful for this purpose. + +From general observation, OpenSeeFace performs well in adverse conditions (low light, high noise, low resolution) and keeps tracking faces through a very wide range of head poses with relatively high stability of landmark positions. Compared to MediaPipe, OpenSeeFace landmarks remain more stable in challenging conditions and it accurately represents a wider range of mouth poses. However, tracking of the eye region can be less accurate. + +I ran OpenSeeFace on a sample clip from the video presentation for [3D Face Reconstruction with Dense Landmarks](https://microsoft.github.io/DenseLandmarks/) by Wood et al. to compare it to MediaPipe and their approach. You can watch the result [here](https://www.vseeface.icu/assets/media/OSFMediaPipe3DFR.mp4). + # Usage A sample Unity project for VRM based avatar animation can be found [here](https://github.com/emilianavt/OpenSeeFaceSample). @@ -30,6 +38,8 @@ A simple demonstration can be achieved by creating a new scene in Unity, adding python facetracker.py --visualize 3 --pnp-points 1 --max-threads 4 -c video.mp4 +__Note__: If dependencies were installed using [poetry](https://python-poetry.org/), the commands have to be executed from a `poetry shell` or have to be prefixed with `poetry run`. + This way the tracking script will output its own tracking visualization while also demonstrating the transmission of tracking data to Unity. The included `OpenSeeLauncher` component allows starting the face tracker program from Unity. It is designed to work with the pyinstaller created executable distributed in the binary release bundles. It provides three public API functions: @@ -91,7 +101,7 @@ Four pretrained face landmark models are included. Using the `--model` switch, i FPS measurements are from running on one core of my CPU. -Pytorch weights for use with `model.py` can be found [here](https://mega.nz/file/vvYXlYQT#h7FpEg4tmOCJNxjpsDEw0JomJIkVGKwrt4OUV0RNDDU). +Pytorch weights for use with `model.py` can be found [here](https://mega.nz/file/vvYXlYQT#h7FpEg4tmOCJNxjpsDEw0JomJIkVGKwrt4OUV0RNDDU). Some unoptimized ONNX models can be found [here](https://github.com/emilianavt/OpenSeeFace/issues/48). # Results @@ -119,9 +129,8 @@ When distributing it, you should also distribute the `Licenses` folder along wit The release builds contain a custom build of ONNX Runtime without telemetry. -# Dependencies +# Dependencies (Python 3.6 - 3.9) -* Python 3.7 * ONNX Runtime * OpenCV * Pillow @@ -131,6 +140,22 @@ The required libraries can be installed using pip: pip install onnxruntime opencv-python pillow numpy +Alternatively poetry can be used to +install all dependencies for this project in a separate virtual env: + + poetry install + +# Dependencies + +* onnxruntime +* OpenCV +* Pillow +* Numpy + +The required libraries can be installed using pip: + + pip install onnxruntime opencv-python pillow numpy + # References ## Training dataset @@ -156,6 +181,8 @@ Additional training has been done on the WFLW dataset after reducing it to 66 po For the training the gaze and blink detection model, the [MPIIGaze](https://www.mpi-inf.mpg.de/departments/computer-vision-and-machine-learning/research/gaze-based-human-computer-interaction/appearance-based-gaze-estimation-in-the-wild/) dataset was used. Additionally, around 125000 synthetic eyes generated with [UnityEyes](https://www.cl.cam.ac.uk/research/rainbow/projects/unityeyes/) were used during training. +It should be noted that additional custom data was also used during the training process and that the reference landmarks from the original datasets have been modified in certain ways to address various issues. It is likely not possible to reproduce these models with just the original LS3D-W and WFLW datasets, however the additional data is not redistributable. + The heatmap regression based face detection model was trained on random 224x224 crops from the WIDER FACE dataset. @inproceedings{yang2016wider, diff --git a/dshowcapture.py b/dshowcapture.py index 46def39..515125c 100644 --- a/dshowcapture.py +++ b/dshowcapture.py @@ -77,7 +77,7 @@ class DShowCapture(): bm_lib.stop_capture_single.argtypes = [] bm_lib.get_json_length.argtypes = [] bm_lib.get_json.argtypes = [c_char_p, c_int] - if not bm_options is None: + if bm_options is not None: bm_lib.set_options.argtypes = [c_char_p] bm_lib.set_options(bm_options.encode()) self.lib = lib @@ -93,7 +93,7 @@ class DShowCapture(): self.type = None def __del__(self): - if not self.buffer is None: + if self.buffer is not None: del self.buffer del self.name_buffer self.destroy_capture() @@ -329,7 +329,7 @@ if __name__ == "__main__": while True: img = cap.get_frame(1000) - if not img is None: + if img is not None: cv2.imshow("DShowCapture", img) if cv2.waitKey(1) & 0xFF == ord('q'): sys.exit(0) diff --git a/dshowcapture/dshowcapture_x64.dll b/dshowcapture/dshowcapture_x64.dll index 88e3752..e7f30fd 100644 Binary files a/dshowcapture/dshowcapture_x64.dll and b/dshowcapture/dshowcapture_x64.dll differ diff --git a/dshowcapture/dshowcapture_x86.dll b/dshowcapture/dshowcapture_x86.dll index 55222da..9eb7308 100644 Binary files a/dshowcapture/dshowcapture_x86.dll and b/dshowcapture/dshowcapture_x86.dll differ diff --git a/facetracker.py b/facetracker.py index 96c4ad3..205b2fb 100644 --- a/facetracker.py +++ b/facetracker.py @@ -13,12 +13,14 @@ if os.name == 'nt': parser.add_argument("-a", "--list-dcaps", type=int, help="Set this to -1 to list all cameras and their available capabilities, set this to a camera id to list that camera's capabilities", default=None) parser.add_argument("-W", "--width", type=int, help="Set camera and raw RGB width", default=640) parser.add_argument("-H", "--height", type=int, help="Set camera and raw RGB height", default=360) - parser.add_argument("-F", "--fps", type=int, help="Set camera frames per second", default=24) - parser.add_argument("-D", "--dcap", type=int, help="Set which device capability line to use or -1 to use the default camera settings", default=None) + parser.add_argument("-D", "--dcap", type=int, help="Set which device capability line to use or -1 to use the default camera settings (FPS still need to be set separately)", default=None) parser.add_argument("-B", "--blackmagic", type=int, help="When set to 1, special support for Blackmagic devices is enabled", default=0) else: parser.add_argument("-W", "--width", type=int, help="Set raw RGB width", default=640) parser.add_argument("-H", "--height", type=int, help="Set raw RGB height", default=360) +if sys.platform == 'linux': + parser.add_argument("--dformat", type=str, help="Set which device format to use (MJPG, YUYV, RGB3, ...)", default=None) +parser.add_argument("-F", "--fps", type=int, help="Set camera frames per second", default=24) parser.add_argument("-c", "--capture", help="Set camera ID (0, 1...) or video file", default="0") parser.add_argument("-M", "--mirror-input", action="store_true", help="Process a mirror image of the input video") parser.add_argument("-m", "--max-threads", type=int, help="Set the maximum number of threads", default=1) @@ -42,7 +44,7 @@ parser.add_argument("--log-data", help="You can set a filename to which tracking parser.add_argument("--log-output", help="You can set a filename to console output will be logged here", default="") parser.add_argument("--model", type=int, help="This can be used to select the tracking model. Higher numbers are models with better tracking quality, but slower speed, except for model 4, which is wink optimized. Models 1 and 0 tend to be too rigid for expression and blink detection. Model -2 is roughly equivalent to model 1, but faster. Model -3 is between models 0 and -1.", default=3, choices=[-3, -2, -1, 0, 1, 2, 3, 4]) parser.add_argument("--model-dir", help="This can be used to specify the path to the directory containing the .onnx model files", default=None) -parser.add_argument("--gaze-tracking", type=int, help="When set to 1, experimental blink detection and gaze tracking are enabled, which makes things slightly slower", default=1) +parser.add_argument("--gaze-tracking", type=int, help="When set to 1, gaze tracking is enabled, which makes things slightly slower", default=1) parser.add_argument("--face-id-offset", type=int, help="When set, this offset is added to all face ids, which can be useful for mixing tracking data from multiple network sources", default=0) parser.add_argument("--repeat-video", type=int, help="When set to 1 and a video file was specified with -c, the tracker will loop the video until interrupted", default=0) parser.add_argument("--dump-points", type=str, help="When set to a filename, the current face 3D points are made symmetric and dumped to the given file when quitting the visualization with the \"q\" key", default="") @@ -60,12 +62,12 @@ class OutputLog(object): self.fh = fh self.output = output def write(self, buf): - if not self.fh is None: + if self.fh is not None: self.fh.write(buf) self.output.write(buf) self.flush() def flush(self): - if not self.fh is None: + if self.fh is not None: self.fh.flush() self.output.flush() output_logfile = None @@ -78,19 +80,19 @@ if os.name == 'nt': import dshowcapture if args.blackmagic == 1: dshowcapture.set_bm_enabled(True) - if not args.blackmagic_options is None: + if args.blackmagic_options is not None: dshowcapture.set_options(args.blackmagic_options) - if not args.priority is None: + if args.priority is not None: import psutil classes = [psutil.IDLE_PRIORITY_CLASS, psutil.BELOW_NORMAL_PRIORITY_CLASS, psutil.NORMAL_PRIORITY_CLASS, psutil.ABOVE_NORMAL_PRIORITY_CLASS, psutil.HIGH_PRIORITY_CLASS, psutil.REALTIME_PRIORITY_CLASS] p = psutil.Process(os.getpid()) p.nice(classes[args.priority]) -if os.name == 'nt' and (args.list_cameras > 0 or not args.list_dcaps is None): +if os.name == 'nt' and (args.list_cameras > 0 or args.list_dcaps is not None): cap = dshowcapture.DShowCapture() info = cap.get_info() unit = 10000000.; - if not args.list_dcaps is None: + if args.list_dcaps is not None: formats = {0: "Any", 1: "Unknown", 100: "ARGB", 101: "XRGB", 200: "I420", 201: "NV12", 202: "YV12", 203: "Y800", 300: "YVYU", 301: "YUY2", 302: "UYVY", 303: "HDYC (Unsupported)", 400: "MJPEG", 401: "H264" } for cam in info: if args.list_dcaps == -1: @@ -153,18 +155,19 @@ target_port = args.port if args.faces >= 40: print("Transmission of tracking data over network is not supported with 40 or more faces.") -fps = 0 +fps = args.fps dcap = None use_dshowcapture_flag = False if os.name == 'nt': - fps = args.fps dcap = args.dcap use_dshowcapture_flag = True if args.use_dshowcapture == 1 else False input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps, use_dshowcapture=use_dshowcapture_flag, dcap=dcap) if args.dcap == -1 and type(input_reader) == DShowCaptureReader: fps = min(fps, input_reader.device.get_fps()) +elif sys.platform == 'linux' and args.dformat: + input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps, dcap=args.dformat) else: - input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps, use_dshowcapture=use_dshowcapture_flag) + input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps) if type(input_reader.reader) == VideoReader: fps = 0 @@ -185,9 +188,9 @@ features = ["eye_l", "eye_r", "eyebrow_steepness_l", "eyebrow_updown_l", "eyebro if args.log_data != "": log = open(args.log_data, "w") log.write("Frame,Time,Width,Height,FPS,Face,FaceID,RightOpen,LeftOpen,AverageConfidence,Success3D,PnPError,RotationQuat.X,RotationQuat.Y,RotationQuat.Z,RotationQuat.W,Euler.X,Euler.Y,Euler.Z,RVec.X,RVec.Y,RVec.Z,TVec.X,TVec.Y,TVec.Z") - for i in range(66): + for i in range(68): log.write(f",Landmark[{i}].X,Landmark[{i}].Y,Landmark[{i}].Confidence") - for i in range(66): + for i in range(70): log.write(f",Point3D[{i}].X,Point3D[{i}].Y,Point3D[{i}].Z") for feature in features: log.write(f",{feature}") @@ -249,7 +252,7 @@ try: height, width, channels = frame.shape sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) tracker = Tracker(width, height, threshold=args.threshold, max_threads=args.max_threads, max_faces=args.faces, discard_after=args.discard_after, scan_every=args.scan_every, silent=False if args.silent == 0 else True, model_type=args.model, model_dir=args.model_dir, no_gaze=False if args.gaze_tracking != 0 and args.model != -1 else True, detection_threshold=args.detection_threshold, use_retinaface=args.scan_retinaface, max_feature_updates=args.max_feature_updates, static_model=True if args.no_3d_adapt == 1 else False, try_hard=args.try_hard == 1) - if not args.video_out is None: + if args.video_out is not None: out = cv2.VideoWriter(args.video_out, cv2.VideoWriter_fourcc('F','F','V','1'), args.video_fps, (width * args.video_scale, height * args.video_scale)) try: @@ -292,7 +295,7 @@ try: packet.extend(bytearray(struct.pack("f", f.translation[0]))) packet.extend(bytearray(struct.pack("f", f.translation[1]))) packet.extend(bytearray(struct.pack("f", f.translation[2]))) - if not log is None: + if log is not None: log.write(f"{frame_count},{now},{width},{height},{fps},{face_num},{f.id},{f.eye_blink[0]},{f.eye_blink[1]},{f.conf},{f.success},{f.pnp_error},{f.quaternion[0]},{f.quaternion[1]},{f.quaternion[2]},{f.quaternion[3]},{f.euler[0]},{f.euler[1]},{f.euler[2]},{f.rotation[0]},{f.rotation[1]},{f.rotation[2]},{f.translation[0]},{f.translation[1]},{f.translation[2]}") for (x,y,c) in f.lms: packet.extend(bytearray(struct.pack("f", c))) @@ -303,32 +306,23 @@ try: for pt_num, (x,y,c) in enumerate(f.lms): packet.extend(bytearray(struct.pack("f", y))) packet.extend(bytearray(struct.pack("f", x))) - if not log is None: + if log is not None: log.write(f",{y},{x},{c}") - if pt_num == 66 and (f.eye_blink[0] < 0.30 or c < 0.30): + if pt_num == 66 and (f.eye_blink[0] < 0.30 or c < 0.20): continue - if pt_num == 67 and (f.eye_blink[1] < 0.30 or c < 0.30): + if pt_num == 67 and (f.eye_blink[1] < 0.30 or c < 0.20): continue x = int(x + 0.5) y = int(y + 0.5) - if args.visualize != 0 or not out is None: + if args.visualize != 0 or out is not None: if args.visualize > 3: frame = cv2.putText(frame, str(pt_num), (int(y), int(x)), cv2.FONT_HERSHEY_SIMPLEX, 0.25, (255,255,0)) color = (0, 255, 0) if pt_num >= 66: color = (255, 255, 0) if not (x < 0 or y < 0 or x >= height or y >= width): - frame[int(x), int(y)] = color - x += 1 - if not (x < 0 or y < 0 or x >= height or y >= width): - frame[int(x), int(y)] = color - y += 1 - if not (x < 0 or y < 0 or x >= height or y >= width): - frame[int(x), int(y)] = color - x -= 1 - if not (x < 0 or y < 0 or x >= height or y >= width): - frame[int(x), int(y)] = color - if args.pnp_points != 0 and (args.visualize != 0 or not out is None) and f.rotation is not None: + cv2.circle(frame, (y, x), 1, color, -1) + if args.pnp_points != 0 and (args.visualize != 0 or out is not None) and f.rotation is not None: if args.pnp_points > 1: projected = cv2.projectPoints(f.face_3d[0:66], f.rotation, f.translation, tracker.camera, tracker.dist_coeffs) else: @@ -351,7 +345,7 @@ try: packet.extend(bytearray(struct.pack("f", x))) packet.extend(bytearray(struct.pack("f", -y))) packet.extend(bytearray(struct.pack("f", -z))) - if not log is None: + if log is not None: log.write(f",{x},{-y},{-z}") if f.current_features is None: f.current_features = {} @@ -359,16 +353,16 @@ try: if not feature in f.current_features: f.current_features[feature] = 0 packet.extend(bytearray(struct.pack("f", f.current_features[feature]))) - if not log is None: + if log is not None: log.write(f",{f.current_features[feature]}") - if not log is None: + if log is not None: log.write("\r\n") log.flush() if detected and len(faces) < 40: sock.sendto(packet, (target_ip, target_port)) - if not out is None: + if out is not None: video_frame = frame if args.video_scale != 1: video_frame = cv2.resize(frame, (width * args.video_scale, height * args.video_scale), interpolation=cv2.INTER_NEAREST) @@ -379,7 +373,7 @@ try: if args.visualize != 0: cv2.imshow('OpenSeeFace Visualization', frame) if cv2.waitKey(1) & 0xFF == ord('q'): - if args.dump_points != "" and not faces is None and len(faces) > 0: + if args.dump_points != "" and faces is not None and len(faces) > 0: np.set_printoptions(threshold=sys.maxsize, precision=15) pairs = [ (0, 16), @@ -455,9 +449,10 @@ except KeyboardInterrupt: print("Quitting") input_reader.close() -if not out is None: +if out is not None: out.release() -cv2.destroyAllWindows() +if args.visualize != 0: + cv2.destroyAllWindows() if args.silent == 0 and tracking_frames > 0: average_tracking_time = 1000 * tracking_time / tracking_frames diff --git a/input_reader.py b/input_reader.py index a62af69..2344ada 100644 --- a/input_reader.py +++ b/input_reader.py @@ -111,6 +111,7 @@ class OpenCVReader(VideoReader): super(OpenCVReader, self).__init__(capture, camera=True) self.cap.set(3, width) self.cap.set(4, height) + self.cap.set(38, 1) def is_open(self): return super(OpenCVReader, self).is_open() def is_ready(self): @@ -120,6 +121,30 @@ class OpenCVReader(VideoReader): def close(self): super(OpenCVReader, self).close() +class V4L2Reader(OpenCVReader): + def __init__(self, capture, width, height, fps, dformat): + self.cap = cv2.VideoCapture(capture, cv2.CAP_V4L2) + if dformat: + fourcc = cv2.VideoWriter.fourcc(*dformat) + self.cap.set(6, fourcc) + self.device = None + self.width = width + self.height = height + self.fps = fps + self.name = str(capture) + self.cap.set(3, width) + self.cap.set(4, height) + self.cap.set(38, 1) + self.cap.set(5, fps) + def is_open(self): + return super(V4L2Reader, self).is_open() + def is_ready(self): + return super(V4L2Reader, self).is_ready() + def read(self): + return super(V4L2Reader, self).read() + def close(self): + super(V4L2Reader, self).close() + class RawReader: def __init__(self, width, height): self.width = int(width) @@ -231,6 +256,14 @@ class InputReader(): print(f"Escapi failed. Falling back to OpenCV. If this fails, please change your camera settings.", file=sys.stderr) self.reader = OpenCVReader(int(capture), width, height, fps) self.name = self.reader.name + elif sys.platform == 'linux': + try: + self.reader = V4L2Reader(int(capture), width, height, fps, dcap) + except: + print("V4L2 exception: ") + traceback.print_exc() + print(f"V4L2 failed. Falling back to OpenCV. If this fails, please change your camera settings.", file=sys.stderr) + self.reader = OpenCVReader(int(capture), width, height, fps) else: self.reader = OpenCVReader(int(capture), width, height, fps) except Exception as e: diff --git a/make_exe.bat b/make_exe.bat index a5ad490..762442b 100644 --- a/make_exe.bat +++ b/make_exe.bat @@ -1,16 +1,31 @@ -pyinstaller facetracker.spec --onedir ^ - --add-binary dshowcapture/dshowcapture_x86.dll;. ^ - --add-binary dshowcapture/dshowcapture_x64.dll;. ^ - --add-binary dshowcapture/libminibmcapture32.dll;. ^ - --add-binary dshowcapture/libminibmcapture64.dll;. ^ - --add-binary escapi/escapi_x86.dll;. ^ - --add-binary escapi/escapi_x64.dll;. ^ +echo "Started" + +:: A venv is used so it's easy to add the onnxruntime dll to the binary + +echo "Creating venv" +python -m venv venv + +echo "Activating venv" +call venv\Scripts\activate.bat + +echo "Installing dependencies" +pip install wheel + +pip install onnxruntime opencv-python==4.5.4.60 pillow numpy==1.23.0 pyinstaller + +echo "Running pyinstaller" +pyinstaller facetracker.py --clean ^ + --onedir ^ + --add-binary dshowcapture/*.dll;. ^ + --add-binary escapi/*.dll;. ^ + --add-binary venv/lib/site-packages/onnxruntime/capi/*.dll;onnxruntime\capi ^ --add-binary msvcp140.dll;. ^ --add-binary vcomp140.dll;. ^ --add-binary concrt140.dll;. ^ --add-binary vccorlib140.dll;. ^ --add-binary run.bat;. -del dist\facetracker\cv2\opencv_videoio_ffmpeg420_64.dll +echo "Deleting opencv dll" +del dist\facetracker\cv2\opencv_videoio_* -pause \ No newline at end of file +echo "Finished" diff --git a/make_exe.sh b/make_exe.sh new file mode 100755 index 0000000..1963f23 --- /dev/null +++ b/make_exe.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +PYTHON_BINARY=$(which python3) + +if [[ -z "$PYTHON_BINARY" ]]; then + echo "python3 not found. Searching for python instead." + PYTHON_BINARY=$(which python) + if [[ -z "$PYTHON_BINARY" ]]; then + echo "Python 3 must be available on your path. Exiting." + exit 1 + fi +fi + +echo "Creating venv" +"$PYTHON_BINARY" -m venv venv + +source venv/bin/activate + +echo "Installing packages" +pip install wheel # Make sure this is installed beforehand + +pip install onnxruntime opencv-python==4.5.4.60 pillow numpy==1.23.0 pyinstaller + +echo "Creating binary" +pyinstaller --onedir --clean facetracker.py \ + --add-binary venv/lib/python3.*/site-packages/onnxruntime/capi/*.so:onnxruntime/capi + +echo "Files should be available in the dist/ folder" +echo "Done!" + diff --git a/model.py b/model.py index 2824779..8f769cd 100644 --- a/model.py +++ b/model.py @@ -1,7 +1,7 @@ # This file is not used by the tracking application and currently outdated import torch import torch.nn as nn -import geffnet.mobilenetv3 # geffnet.mobilenetv3._gen_mobilenet_v3 needs to be patched to return the parameters instead of instantiating the network +import geffnet.mobilenetv3 # geffnet.mobilenetv3._gen_mobilenet_v3 needs to be patched to return the parameters instead of instantiating the network, based on commit c450c12ae6ffb1757f62dde3c2765da3c10f6def from geffnet.efficientnet_builder import round_channels class DSConv2d(nn.Module): @@ -251,7 +251,7 @@ def AdapWingLoss(pre_hm, gt_hm): img_dilate = cv2.morphologyEx(img_merge, cv2.MORPH_DILATE, kernel) img_dilate[img_dilate < 51] = 1 # 0*W+1 img_dilate[img_dilate >= 51] = 11 # 1*W+1 - img_dilate = np.array(img_dilate, dtype=np.int) + img_dilate = np.array(img_dilate, dtype=int) img_dilate = img_dilate.transpose(2, 0, 1) dilated = torch.from_numpy(img_dilate).float() if first: diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..ffcd882 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,224 @@ +[[package]] +name = "flatbuffers" +version = "2.0" +description = "The FlatBuffers serialization format for Python" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "numpy" +version = "1.22" +description = "NumPy is the fundamental package for array computing with Python." +category = "main" +optional = false +python-versions = ">=3.7,<3.11" + +[[package]] +name = "onnxruntime" +version = "1.9.0" +description = "ONNX Runtime is a runtime accelerator for Machine Learning models" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +flatbuffers = "*" +numpy = ">=1.16.6" +protobuf = "*" + +[[package]] +name = "opencv-python" +version = "4.5.4.58" +description = "Wrapper package for OpenCV python bindings." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +numpy = ">=1.21.2" + +[[package]] +name = "pillow" +version = "10.2.0" +description = "Python Imaging Library (Fork)" +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "protobuf" +version = "4.25.8" +description = "Protocol Buffers" +category = "main" +optional = false +python-versions = ">=3.5" + +[metadata] +lock-version = "1.1" +python-versions = ">=3.7,<3.11" +content-hash = "1511681290c580145b1117beffe728052d856eee3ad8419930626c6fc6c9a09d" + +[metadata.files] +flatbuffers = [ + {file = "flatbuffers-2.0-py2.py3-none-any.whl", hash = "sha256:3751954f0604580d3219ae49a85fafec9d85eec599c0b96226e1bc0b48e57474"}, + {file = "flatbuffers-2.0.tar.gz", hash = "sha256:12158ab0272375eab8db2d663ae97370c33f152b27801fa6024e1d6105fd4dd2"}, +] +numpy = [ + {file = "numpy-1.21.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:508b0b513fa1266875524ba8a9ecc27b02ad771fe1704a16314dc1a816a68737"}, + {file = "numpy-1.21.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5dfe9d6a4c39b8b6edd7990091fea4f852888e41919d0e6722fe78dd421db0eb"}, + {file = "numpy-1.21.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a10968963640e75cc0193e1847616ab4c718e83b6938ae74dea44953950f6b7"}, + {file = "numpy-1.21.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49c6249260890e05b8111ebfc391ed58b3cb4b33e63197b2ec7f776e45330721"}, + {file = "numpy-1.21.3-cp310-cp310-win_amd64.whl", hash = "sha256:f8f4625536926a155b80ad2bbff44f8cc59e9f2ad14cdda7acf4c135b4dc8ff2"}, + {file = "numpy-1.21.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e54af82d68ef8255535a6cdb353f55d6b8cf418a83e2be3569243787a4f4866f"}, + {file = "numpy-1.21.3-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f41b018f126aac18583956c54544db437f25c7ee4794bcb23eb38bef8e5e192a"}, + {file = "numpy-1.21.3-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:50cd26b0cf6664cb3b3dd161ba0a09c9c1343db064e7c69f9f8b551f5104d654"}, + {file = "numpy-1.21.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cc9b512e9fb590797474f58b7f6d1f1b654b3a94f4fa8558b48ca8b3cfc97cf"}, + {file = "numpy-1.21.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:88a5d6b268e9ad18f3533e184744acdaa2e913b13148160b1152300c949bbb5f"}, + {file = "numpy-1.21.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:3c09418a14471c7ae69ba682e2428cae5b4420a766659605566c0fa6987f6b7e"}, + {file = "numpy-1.21.3-cp37-cp37m-win32.whl", hash = "sha256:90bec6a86b348b4559b6482e2b684db4a9a7eed1fa054b86115a48d58fbbf62a"}, + {file = "numpy-1.21.3-cp37-cp37m-win_amd64.whl", hash = "sha256:043e83bfc274649c82a6f09836943e4a4aebe5e33656271c7dbf9621dd58b8ec"}, + {file = "numpy-1.21.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:75621882d2230ab77fb6a03d4cbccd2038511491076e7964ef87306623aa5272"}, + {file = "numpy-1.21.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:188031f833bbb623637e66006cf75e933e00e7231f67e2b45cf8189612bb5dc3"}, + {file = "numpy-1.21.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:160ccc1bed3a8371bf0d760971f09bfe80a3e18646620e9ded0ad159d9749baa"}, + {file = "numpy-1.21.3-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:29fb3dcd0468b7715f8ce2c0c2d9bbbaf5ae686334951343a41bd8d155c6ea27"}, + {file = "numpy-1.21.3-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:32437f0b275c1d09d9c3add782516413e98cd7c09e6baf4715cbce781fc29912"}, + {file = "numpy-1.21.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e606e6316911471c8d9b4618e082635cfe98876007556e89ce03d52ff5e8fcf0"}, + {file = "numpy-1.21.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a99a6b067e5190ac6d12005a4d85aa6227c5606fa93211f86b1dafb16233e57d"}, + {file = "numpy-1.21.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:dde972a1e11bb7b702ed0e447953e7617723760f420decb97305e66fb4afc54f"}, + {file = "numpy-1.21.3-cp38-cp38-win32.whl", hash = "sha256:fe52dbe47d9deb69b05084abd4b0df7abb39a3c51957c09f635520abd49b29dd"}, + {file = "numpy-1.21.3-cp38-cp38-win_amd64.whl", hash = "sha256:75eb7cadc8da49302f5b659d40ba4f6d94d5045fbd9569c9d058e77b0514c9e4"}, + {file = "numpy-1.21.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2a6ee9620061b2a722749b391c0d80a0e2ae97290f1b32e28d5a362e21941ee4"}, + {file = "numpy-1.21.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5c4193f70f8069550a1788bd0cd3268ab7d3a2b70583dfe3b2e7f421e9aace06"}, + {file = "numpy-1.21.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:28f15209fb535dd4c504a7762d3bc440779b0e37d50ed810ced209e5cea60d96"}, + {file = "numpy-1.21.3-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c6c2d535a7beb1f8790aaa98fd089ceab2e3dd7ca48aca0af7dc60e6ef93ffe1"}, + {file = "numpy-1.21.3-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:bffa2eee3b87376cc6b31eee36d05349571c236d1de1175b804b348dc0941e3f"}, + {file = "numpy-1.21.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc14e7519fab2a4ed87d31f99c31a3796e4e1fe63a86ebdd1c5a1ea78ebd5896"}, + {file = "numpy-1.21.3-cp39-cp39-win32.whl", hash = "sha256:dd0482f3fc547f1b1b5d6a8b8e08f63fdc250c58ce688dedd8851e6e26cff0f3"}, + {file = "numpy-1.21.3-cp39-cp39-win_amd64.whl", hash = "sha256:300321e3985c968e3ae7fbda187237b225f3ffe6528395a5b7a5407f73cf093e"}, + {file = "numpy-1.21.3-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98339aa9911853f131de11010f6dd94c8cec254d3d1f7261528c3b3e3219f139"}, + {file = "numpy-1.21.3.zip", hash = "sha256:63571bb7897a584ca3249c86dd01c10bcb5fe4296e3568b2e9c1a55356b6410e"}, +] +onnxruntime = [ + {file = "onnxruntime-1.9.0-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:48f0fcf3c9aa6836584e64abe63fa7395c02066d3259bbdeb489b4d172e0127a"}, + {file = "onnxruntime-1.9.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f9d772a6330cb85e7723f84e357320a1603e3824a92aab4ef36fc3a41e64f16"}, + {file = "onnxruntime-1.9.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3bdb861822a63404cca7b46dce86d48bbc21c906a4b4ed13969bc89763ac7f96"}, + {file = "onnxruntime-1.9.0-cp36-cp36m-win32.whl", hash = "sha256:e3f8f7d5d4d66e3a4a2b731a000d3142a53a5403e8814e68bbd659514e815899"}, + {file = "onnxruntime-1.9.0-cp36-cp36m-win_amd64.whl", hash = "sha256:cf3edbc54bfe99a119d73cd65398a2ec68ae3af2557ab7e645976314a8d11aa1"}, + {file = "onnxruntime-1.9.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:c8ff9c914b2b1c3b022dedc199e3f971e340d8923a1ef42d66530508fa367bf6"}, + {file = "onnxruntime-1.9.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d03ff4a2717c4149acc7c649fd66a67e81ec44c9e6e2a00df1d6e9ca843f1b7"}, + {file = "onnxruntime-1.9.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8a2315e2244ac371742f6e30da5367c680c3e84c31e291a35f8ddfab09c3c82"}, + {file = "onnxruntime-1.9.0-cp37-cp37m-win32.whl", hash = "sha256:fa927b1825f2851c0c8f3948515a56d76cb0686da9acd1d6f8fafe552c8d8fec"}, + {file = "onnxruntime-1.9.0-cp37-cp37m-win_amd64.whl", hash = "sha256:9ccaf6a0365f2b86efe21681416b8cfe97f084a7d53bd1cf2bf889a0aef2b0d3"}, + {file = "onnxruntime-1.9.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:d20ce3448babe89a77cc9d357730767deb3617e36439bddcd006f28abc72b416"}, + {file = "onnxruntime-1.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3fd1d6647245aa38e1099cfd355d84e807de5350d5216e84ceefd91c64ce243d"}, + {file = "onnxruntime-1.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f00620fc0f51bc4d90ae6d96ceb4b6538e3bd1e328178104118ac672f37c40d"}, + {file = "onnxruntime-1.9.0-cp38-cp38-win32.whl", hash = "sha256:e1c1fe3f7d960eeffc02a5f196d85529254eefd59cbeecd8abee0a9467b5c2d8"}, + {file = "onnxruntime-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:9002214af1b2317ab3a63a2f045f7d1363c207e661d475a877aa6499ca09d606"}, + {file = "onnxruntime-1.9.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:7339cef9b918b88f1fec8109cfa0a8416f119c5968d00300a9186847d86e35de"}, + {file = "onnxruntime-1.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdec8538eb59e63a376d0677f7ec043ceb597d52ee88f1f7e250928893a0de7f"}, + {file = "onnxruntime-1.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab62b29429e0e62c11478b2a8a3af2646531fba7800736e8b201d8baa50b43a8"}, + {file = "onnxruntime-1.9.0-cp39-cp39-win32.whl", hash = "sha256:4aee9a893f93637341fd0e6b56fa3ab1c430d718d08d79a358603297a1575ad9"}, + {file = "onnxruntime-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:bdf1327932227383b04093a51266474b2703b3fcf9c0f6f11c652d9652b76a5c"}, +] +opencv-python = [ + {file = "opencv-python-4.5.4.58.tar.gz", hash = "sha256:48288428f407bacba5f73d460feb4a1ecafe87db3d7cfc0730a49fb32f589bbf"}, + {file = "opencv_python-4.5.4.58-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0eba0bfe62c48a02a5af3a0944e872c99f57f98653bed14d51c6991a58f9e1d1"}, + {file = "opencv_python-4.5.4.58-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:9bcca50c5444b5cfb01624666b69f91ba8f2d2bf4ef37b111697aafdeb81c99f"}, + {file = "opencv_python-4.5.4.58-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:8f7886acabaebf0361bd3dbccaa0d08e3f65ab13b7c739eb11e028f01ad13582"}, + {file = "opencv_python-4.5.4.58-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:d4b1d0b98ee72ba5dd720166790fc93ce459281e138ee79b0d41420b3da52b2e"}, + {file = "opencv_python-4.5.4.58-cp310-cp310-win32.whl", hash = "sha256:69a78e40a374ac14e4bf15a13dbb6c30fd2fbd5fcd3674d020a31b88861d5aaf"}, + {file = "opencv_python-4.5.4.58-cp310-cp310-win_amd64.whl", hash = "sha256:315c357522b6310ef7a0718d9f0c5d3110e59c19140705499a3c29bdd8c0124f"}, + {file = "opencv_python-4.5.4.58-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:887a61097092dc0bf23fa24646dbc8cfeeb753649cb28a3782a93a6879e3b7d2"}, + {file = "opencv_python-4.5.4.58-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:22bcc3153a7d4f95aff79457eef81ef5e40ab1851b189e014412b5e9fbee2573"}, + {file = "opencv_python-4.5.4.58-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:92e9b2261ec764229c948d77fe0d922ee033348ca6519939b87861016c1614b3"}, + {file = "opencv_python-4.5.4.58-cp36-cp36m-win32.whl", hash = "sha256:0d6249a49122a78afc6685ddb1377a87e46414ae61c84535c4c6024397f1f3e8"}, + {file = "opencv_python-4.5.4.58-cp36-cp36m-win_amd64.whl", hash = "sha256:eaa144013b597e4dcabc8d8230edfe810319de01b5609556d415a20e2b707547"}, + {file = "opencv_python-4.5.4.58-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:26feeeb280de179f5dbb8976ebf7ceb836bd263973cb5daec8ca36e8ef7b5773"}, + {file = "opencv_python-4.5.4.58-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:4a13381bdfc0fb4b080efcc27c46561d0bd752f126226e9f19aa9cbcf6677f40"}, + {file = "opencv_python-4.5.4.58-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:ac852fcaac93439f2f7116ddffdc23fd366c872200ade2272446f9898180cecb"}, + {file = "opencv_python-4.5.4.58-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:02872e0a9358526646d691f390143e9c21109c210095314abaa0641211cda077"}, + {file = "opencv_python-4.5.4.58-cp37-cp37m-win32.whl", hash = "sha256:6b87bab220d17e03eeedbcc6652d9d7e7bb09886dbd0f810310697a948b4c6fd"}, + {file = "opencv_python-4.5.4.58-cp37-cp37m-win_amd64.whl", hash = "sha256:a2a7f09b8843b85f3e1b02c5ea3ddc0cb9f5ad9698380109b37069ee8db7746d"}, + {file = "opencv_python-4.5.4.58-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:c44f5c51e92322ed832607204249c190764dec6cf29e8ba6d679b10326be1c1b"}, + {file = "opencv_python-4.5.4.58-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9b2c198af083a693d42a82bddc4d1f7e6bb02c64192ff7fac1fd1d43a8cf1be6"}, + {file = "opencv_python-4.5.4.58-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:637f4d3ad81bd27f273ede4c5fa6c26afb85c097c9715baf107cc270e37f5fea"}, + {file = "opencv_python-4.5.4.58-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:2fff48a641a74d1def31c1e88f9e5ce50ba4d0f87d085dfbf8bc844e12f6cd54"}, + {file = "opencv_python-4.5.4.58-cp38-cp38-win32.whl", hash = "sha256:8ddf4dcd8199209e33f21deb0c6d8ab62b21802816bba895fefc346b6d2e522d"}, + {file = "opencv_python-4.5.4.58-cp38-cp38-win_amd64.whl", hash = "sha256:085c5fcf5a6479c34aca3fd0f59055e704083d6a44009d6583c675ff1a5a0625"}, + {file = "opencv_python-4.5.4.58-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:4abe9c4fb6fe16daa9fcdd68b5357d3530431341aa655203f8e84f394e1fe6d4"}, + {file = "opencv_python-4.5.4.58-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4b614fbd81aeda53ce28e645aaee18fda7c7f2a48eb7f1a70a7c6c3427946342"}, + {file = "opencv_python-4.5.4.58-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:215bdf069847d4e3b0447a34e9eb4046dd4ca523d41fe4381c1c55f6704fd0dc"}, + {file = "opencv_python-4.5.4.58-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc34cdbfbab463750713118c8259a5d364547adab8ed91e94ba888349f33590a"}, + {file = "opencv_python-4.5.4.58-cp39-cp39-win32.whl", hash = "sha256:9998ce60884f3cda074f02b56d2b57ee6bd863e2ddba132da2b0af3b9487d584"}, + {file = "opencv_python-4.5.4.58-cp39-cp39-win_amd64.whl", hash = "sha256:5370a11757fbe94b176771269aff599f4da8676c2a672b13bcbca043f2e3eea8"}, +] +pillow = [ + {file = "Pillow-8.4.0-cp310-cp310-macosx_10_10_universal2.whl", hash = "sha256:81f8d5c81e483a9442d72d182e1fb6dcb9723f289a57e8030811bac9ea3fef8d"}, + {file = "Pillow-8.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3f97cfb1e5a392d75dd8b9fd274d205404729923840ca94ca45a0af57e13dbe6"}, + {file = "Pillow-8.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb9fc393f3c61f9054e1ed26e6fe912c7321af2f41ff49d3f83d05bacf22cc78"}, + {file = "Pillow-8.4.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d82cdb63100ef5eedb8391732375e6d05993b765f72cb34311fab92103314649"}, + {file = "Pillow-8.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:62cc1afda735a8d109007164714e73771b499768b9bb5afcbbee9d0ff374b43f"}, + {file = "Pillow-8.4.0-cp310-cp310-win32.whl", hash = "sha256:e3dacecfbeec9a33e932f00c6cd7996e62f53ad46fbe677577394aaa90ee419a"}, + {file = "Pillow-8.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:620582db2a85b2df5f8a82ddeb52116560d7e5e6b055095f04ad828d1b0baa39"}, + {file = "Pillow-8.4.0-cp36-cp36m-macosx_10_10_x86_64.whl", hash = "sha256:1bc723b434fbc4ab50bb68e11e93ce5fb69866ad621e3c2c9bdb0cd70e345f55"}, + {file = "Pillow-8.4.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:72cbcfd54df6caf85cc35264c77ede902452d6df41166010262374155947460c"}, + {file = "Pillow-8.4.0-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:70ad9e5c6cb9b8487280a02c0ad8a51581dcbbe8484ce058477692a27c151c0a"}, + {file = "Pillow-8.4.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25a49dc2e2f74e65efaa32b153527fc5ac98508d502fa46e74fa4fd678ed6645"}, + {file = "Pillow-8.4.0-cp36-cp36m-win32.whl", hash = "sha256:93ce9e955cc95959df98505e4608ad98281fff037350d8c2671c9aa86bcf10a9"}, + {file = "Pillow-8.4.0-cp36-cp36m-win_amd64.whl", hash = "sha256:2e4440b8f00f504ee4b53fe30f4e381aae30b0568193be305256b1462216feff"}, + {file = "Pillow-8.4.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:8c803ac3c28bbc53763e6825746f05cc407b20e4a69d0122e526a582e3b5e153"}, + {file = "Pillow-8.4.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c8a17b5d948f4ceeceb66384727dde11b240736fddeda54ca740b9b8b1556b29"}, + {file = "Pillow-8.4.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1394a6ad5abc838c5cd8a92c5a07535648cdf6d09e8e2d6df916dfa9ea86ead8"}, + {file = "Pillow-8.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:792e5c12376594bfcb986ebf3855aa4b7c225754e9a9521298e460e92fb4a488"}, + {file = "Pillow-8.4.0-cp37-cp37m-win32.whl", hash = "sha256:d99ec152570e4196772e7a8e4ba5320d2d27bf22fdf11743dd882936ed64305b"}, + {file = "Pillow-8.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:7b7017b61bbcdd7f6363aeceb881e23c46583739cb69a3ab39cb384f6ec82e5b"}, + {file = "Pillow-8.4.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:d89363f02658e253dbd171f7c3716a5d340a24ee82d38aab9183f7fdf0cdca49"}, + {file = "Pillow-8.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0a0956fdc5defc34462bb1c765ee88d933239f9a94bc37d132004775241a7585"}, + {file = "Pillow-8.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b7bb9de00197fb4261825c15551adf7605cf14a80badf1761d61e59da347779"}, + {file = "Pillow-8.4.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:72b9e656e340447f827885b8d7a15fc8c4e68d410dc2297ef6787eec0f0ea409"}, + {file = "Pillow-8.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5a4532a12314149d8b4e4ad8ff09dde7427731fcfa5917ff16d0291f13609df"}, + {file = "Pillow-8.4.0-cp38-cp38-win32.whl", hash = "sha256:82aafa8d5eb68c8463b6e9baeb4f19043bb31fefc03eb7b216b51e6a9981ae09"}, + {file = "Pillow-8.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:066f3999cb3b070a95c3652712cffa1a748cd02d60ad7b4e485c3748a04d9d76"}, + {file = "Pillow-8.4.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:5503c86916d27c2e101b7f71c2ae2cddba01a2cf55b8395b0255fd33fa4d1f1a"}, + {file = "Pillow-8.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4acc0985ddf39d1bc969a9220b51d94ed51695d455c228d8ac29fcdb25810e6e"}, + {file = "Pillow-8.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b052a619a8bfcf26bd8b3f48f45283f9e977890263e4571f2393ed8898d331b"}, + {file = "Pillow-8.4.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:493cb4e415f44cd601fcec11c99836f707bb714ab03f5ed46ac25713baf0ff20"}, + {file = "Pillow-8.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8831cb7332eda5dc89b21a7bce7ef6ad305548820595033a4b03cf3091235ed"}, + {file = "Pillow-8.4.0-cp39-cp39-win32.whl", hash = "sha256:5e9ac5f66616b87d4da618a20ab0a38324dbe88d8a39b55be8964eb520021e02"}, + {file = "Pillow-8.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:3eb1ce5f65908556c2d8685a8f0a6e989d887ec4057326f6c22b24e8a172c66b"}, + {file = "Pillow-8.4.0-pp36-pypy36_pp73-macosx_10_10_x86_64.whl", hash = "sha256:ddc4d832a0f0b4c52fff973a0d44b6c99839a9d016fe4e6a1cb8f3eea96479c2"}, + {file = "Pillow-8.4.0-pp36-pypy36_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9a3e5ddc44c14042f0844b8cf7d2cd455f6cc80fd7f5eefbe657292cf601d9ad"}, + {file = "Pillow-8.4.0-pp36-pypy36_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c70e94281588ef053ae8998039610dbd71bc509e4acbc77ab59d7d2937b10698"}, + {file = "Pillow-8.4.0-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:3862b7256046fcd950618ed22d1d60b842e3a40a48236a5498746f21189afbbc"}, + {file = "Pillow-8.4.0-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a4901622493f88b1a29bd30ec1a2f683782e57c3c16a2dbc7f2595ba01f639df"}, + {file = "Pillow-8.4.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84c471a734240653a0ec91dec0996696eea227eafe72a33bd06c92697728046b"}, + {file = "Pillow-8.4.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:244cf3b97802c34c41905d22810846802a3329ddcb93ccc432870243211c79fc"}, + {file = "Pillow-8.4.0.tar.gz", hash = "sha256:b8e2f83c56e141920c39464b852de3719dfbfb6e3c99a2d8da0edf4fb33176ed"}, +] +protobuf = [ + {file = "protobuf-3.19.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d80f80eb175bf5f1169139c2e0c5ada98b1c098e2b3c3736667f28cbbea39fc8"}, + {file = "protobuf-3.19.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:a529e7df52204565bcd33738a7a5f288f3d2d37d86caa5d78c458fa5fabbd54d"}, + {file = "protobuf-3.19.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28ccea56d4dc38d35cd70c43c2da2f40ac0be0a355ef882242e8586c6d66666f"}, + {file = "protobuf-3.19.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8b30a7de128c46b5ecb343917d9fa737612a6e8280f440874e5cc2ba0d79b8f6"}, + {file = "protobuf-3.19.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5935c8ce02e3d89c7900140a8a42b35bc037ec07a6aeb61cc108be8d3c9438a6"}, + {file = "protobuf-3.19.1-cp36-cp36m-win32.whl", hash = "sha256:74f33edeb4f3b7ed13d567881da8e5a92a72b36495d57d696c2ea1ae0cfee80c"}, + {file = "protobuf-3.19.1-cp36-cp36m-win_amd64.whl", hash = "sha256:038daf4fa38a7e818dd61f51f22588d61755160a98db087a046f80d66b855942"}, + {file = "protobuf-3.19.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e51561d72efd5bd5c91490af1f13e32bcba8dab4643761eb7de3ce18e64a853"}, + {file = "protobuf-3.19.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:6e8ea9173403219239cdfd8d946ed101f2ab6ecc025b0fda0c6c713c35c9981d"}, + {file = "protobuf-3.19.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db3532d9f7a6ebbe2392041350437953b6d7a792de10e629c1e4f5a6b1fe1ac6"}, + {file = "protobuf-3.19.1-cp37-cp37m-win32.whl", hash = "sha256:615b426a177780ce381ecd212edc1e0f70db8557ed72560b82096bd36b01bc04"}, + {file = "protobuf-3.19.1-cp37-cp37m-win_amd64.whl", hash = "sha256:d8919368410110633717c406ab5c97e8df5ce93020cfcf3012834f28b1fab1ea"}, + {file = "protobuf-3.19.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:71b0250b0cfb738442d60cab68abc166de43411f2a4f791d31378590bfb71bd7"}, + {file = "protobuf-3.19.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:3cd0458870ea7d1c58e948ac8078f6ba8a7ecc44a57e03032ed066c5bb318089"}, + {file = "protobuf-3.19.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:655264ed0d0efe47a523e2255fc1106a22f6faab7cc46cfe99b5bae085c2a13e"}, + {file = "protobuf-3.19.1-cp38-cp38-win32.whl", hash = "sha256:b691d996c6d0984947c4cf8b7ae2fe372d99b32821d0584f0b90277aa36982d3"}, + {file = "protobuf-3.19.1-cp38-cp38-win_amd64.whl", hash = "sha256:e7e8d2c20921f8da0dea277dfefc6abac05903ceac8e72839b2da519db69206b"}, + {file = "protobuf-3.19.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fd390367fc211cc0ffcf3a9e149dfeca78fecc62adb911371db0cec5c8b7472d"}, + {file = "protobuf-3.19.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d83e1ef8cb74009bebee3e61cc84b1c9cd04935b72bca0cbc83217d140424995"}, + {file = "protobuf-3.19.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36d90676d6f426718463fe382ec6274909337ca6319d375eebd2044e6c6ac560"}, + {file = "protobuf-3.19.1-cp39-cp39-win32.whl", hash = "sha256:e7b24c11df36ee8e0c085e5b0dc560289e4b58804746fb487287dda51410f1e2"}, + {file = "protobuf-3.19.1-cp39-cp39-win_amd64.whl", hash = "sha256:77d2fadcf369b3f22859ab25bd12bb8e98fb11e05d9ff9b7cd45b711c719c002"}, + {file = "protobuf-3.19.1-py2.py3-none-any.whl", hash = "sha256:e813b1c9006b6399308e917ac5d298f345d95bb31f46f02b60cd92970a9afa17"}, + {file = "protobuf-3.19.1.tar.gz", hash = "sha256:62a8e4baa9cb9e064eb62d1002eca820857ab2138440cb4b3ea4243830f94ca7"}, +] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..de1cc7b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,19 @@ +[tool.poetry] +name = "openseeface" +version = "1.20.4" +description = "" +authors = ["emilianavt"] +license = "BSD-2-Clause" + +[tool.poetry.dependencies] +python = ">=3.7,<3.11" +numpy = "^1.21.3" +opencv-python = "^4.5.4" +Pillow = "^8.4.0" +onnxruntime = "^1.9.0" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/retinaface.py b/retinaface.py index c97c957..b1d1ae5 100644 --- a/retinaface.py +++ b/retinaface.py @@ -66,7 +66,8 @@ class RetinaFaceDetector(): options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL options.log_severity_level = 3 - self.session = onnxruntime.InferenceSession(model_path, sess_options=options) + providersList = onnxruntime.capi._pybind_state.get_available_providers() + self.session = onnxruntime.InferenceSession(model_path, sess_options=options, providers=providersList) self.res_w, self.res_h = res with open(json_path, "r") as prior_file: self.priorbox = np.array(json.loads(prior_file.read())) diff --git a/tracker.py b/tracker.py index a22190b..26f8305 100644 --- a/tracker.py +++ b/tracker.py @@ -140,7 +140,7 @@ def worker_thread(session, frame, input, crop_info, queue, input_name, idx, trac conf, lms = tracker.landmarks(output[0], crop_info) if conf > tracker.threshold: try: - eye_state = tracker.get_eye_state(frame, lms, single=True) + eye_state = tracker.get_eye_state(frame, lms) except: eye_state = [(1.0, 0.0, 0.0, 0.0), (1.0, 0.0, 0.0, 0.0)] queue.put((session, conf, (lms, eye_state), crop_info, idx)) @@ -527,8 +527,12 @@ class Tracker(): self.retinaface_scan = RetinaFaceDetector(model_path=os.path.join(model_base_path, "retinaface_640x640_opt.onnx"), json_path=os.path.join(model_base_path, "priorbox_640x640.json"), threads=2, top_k=max_faces, res=(640, 640)) self.use_retinaface = use_retinaface + # OTR 1.9 and later requires specifying providers + # explicitly as an argument in InferenceSession() + providersList = onnxruntime.capi._pybind_state.get_available_providers() + # Single face instance with multiple threads - self.session = onnxruntime.InferenceSession(os.path.join(model_base_path, model), sess_options=options) + self.session = onnxruntime.InferenceSession(os.path.join(model_base_path, model), sess_options=options, providers=providersList) # Multiple faces with single threads self.sessions = [] @@ -544,20 +548,18 @@ class Tracker(): options.intra_op_num_threads += 1 options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL - self.sessions.append(onnxruntime.InferenceSession(os.path.join(model_base_path, model), sess_options=options)) + self.sessions.append(onnxruntime.InferenceSession(os.path.join(model_base_path, model), sess_options=options, providers=providersList)) self.input_name = self.session.get_inputs()[0].name options = onnxruntime.SessionOptions() options.inter_op_num_threads = 1 - options.intra_op_num_threads = max(max_threads,4) + options.intra_op_num_threads = 1 options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL options.log_severity_level = 3 - self.gaze_model = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options) - options.intra_op_num_threads = 1 - self.gaze_model_single = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options) + self.gaze_model = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options, providers=providersList) - self.detection = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_detection_opt.onnx"), sess_options=options) + self.detection = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_detection_opt.onnx"), sess_options=options, providers=providersList) self.faces = [] # Image normalization constants @@ -760,12 +762,12 @@ class Tracker(): return (avg_conf, np.array(lms)) def estimate_depth(self, face_info): - lms = np.concatenate((face_info.lms, np.array([[face_info.eye_state[0][1], face_info.eye_state[0][2], face_info.eye_state[0][3]], [face_info.eye_state[1][1], face_info.eye_state[1][2], face_info.eye_state[1][3]]], np.float)), 0) + lms = np.concatenate((face_info.lms, np.array([[face_info.eye_state[0][1], face_info.eye_state[0][2], face_info.eye_state[0][3]], [face_info.eye_state[1][1], face_info.eye_state[1][2], face_info.eye_state[1][3]]], np.float32)), 0) image_pts = np.array(lms)[face_info.contour_pts, 0:2] success = False - if not face_info.rotation is None: + if face_info.rotation is not None: success, face_info.rotation, face_info.translation = cv2.solvePnP(face_info.contour, image_pts, self.camera, self.dist_coeffs, useExtrinsicGuess=True, rvec=np.transpose(face_info.rotation), tvec=np.transpose(face_info.translation), flags=cv2.SOLVEPNP_ITERATIVE) else: rvec = np.array([0, 0, 0], np.float32) @@ -781,8 +783,10 @@ class Tracker(): face_info.translation = np.array([0.0, 0.0, 0.0], np.float32) return False, np.zeros(4), np.zeros(3), 99999., pts_3d, lms else: - face_info.rotation = np.transpose(face_info.rotation) - face_info.translation = np.transpose(face_info.translation) + # OpenCV returns (3,1); flatten so [0]/[1]/[2] are scalars for UDP packing + # and 1D .dot() math does not broadcast to (1,3). + face_info.rotation = np.asarray(face_info.rotation, dtype=np.float32).reshape(3) + face_info.translation = np.asarray(face_info.translation, dtype=np.float32).reshape(3) rmat, _ = cv2.Rodrigues(rotation) inverse_rotation = np.linalg.inv(rmat) @@ -831,7 +835,7 @@ class Tracker(): reference = rmat.dot(pt) reference = reference + face_info.translation reference = self.camera.dot(reference) - depth = reference[2] + depth = float(reference[2]) pt_3d = np.array([lms[66+i][0] * depth, lms[66+i][1] * depth, depth], np.float32) pt_3d = self.inverse_camera.dot(pt_3d) pt_3d = pt_3d - face_info.translation @@ -918,11 +922,11 @@ class Tracker(): x1, y1 = clamp_to_im(center - radius, h, w) x2, y2 = clamp_to_im(center + radius + 1, h, w) offset = np.array((x1, y1)) - lms = (lms[:, 0:2] - offset).astype(np.int) + lms = (lms[:, 0:2] - offset).astype(int) frame = frame[y1:y2, x1:x2] return frame, lms, offset - def get_eye_state(self, frame, lms, single=False): + def get_eye_state(self, frame, lms): if self.no_gaze: return [(1.0, 0.0, 0.0, 0.0), (1.0, 0.0, 0.0, 0.0)] lms = np.array(lms) @@ -938,10 +942,7 @@ class Tracker(): return [(1.0, 0.0, 0.0, 0.0), (1.0, 0.0, 0.0, 0.0)] both_eyes = np.concatenate((right_eye, left_eye)) results = None - if single: - results = self.gaze_model_single.run([], {self.input_name: both_eyes}) - else: - results = self.gaze_model.run([], {self.input_name: both_eyes}) + results = self.gaze_model.run([], {self.input_name: both_eyes}) open = [0, 0] open[0] = 1#results[1][0].argmax() open[1] = 1#results[1][1].argmax()