Fix PnP rotation/translation shape for modern OpenCV/NumPy.

solvePnP returns (3,1); transpose made (1,3), which broke eye depth .dot()
and struct.pack of translation scalars. Flatten to (3,) instead.

Co-authored-by: Cursor <cursoragent@cursor.com>

.github/actions/upload-dated-artifact/action.yaml

@@ -0,0 +1,26 @@
+name: Upload dated artifact
+description: Uploads an artifact with a datetime
+
+inputs:
+  name:
+    description: The name of the artifact
+    required: true
+  path:
+    description: The path to the file/folder to upload
+    required: true
+
+runs:
+  using: composite
+  steps:
+    # https://stackoverflow.com/questions/60942067/get-current-date-and-time-in-github-workflows
+    - name: Get current datetime
+      id: date
+      shell: bash
+      run: echo "::set-output name=date::$(date '+%Y-%m-%d_%H-%M-%S')"
+
+    - name: Upload artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: ${{ inputs.name }}_${{ steps.date.outputs.date }}
+        path: ${{ inputs.path }}
+        if-no-files-found: error

.github/workflows/create-binaries.yaml

@@ -0,0 +1,87 @@
+name: Package using PyInstaller
+on:
+  push:
+    tags:
+      - v**
+  workflow_dispatch:
+
+jobs:
+  build-windows:
+    env:
+      dist-path: dist/OpenSeeFace
+    runs-on: windows-latest
+    steps:
+      - name: Pull OpenSeeFace
+        uses: actions/checkout@v3
+
+      - name: Create binary
+        shell: cmd
+        run: |
+          choco install vcredist140
+
+          xcopy c:\Windows\System32\msvcp140.dll .
+          xcopy c:\Windows\System32\vcomp140.dll .
+          xcopy c:\Windows\System32\concrt140.dll .
+          xcopy c:\Windows\System32\vccorlib140.dll .
+
+          make_exe.bat
+
+      - name: Move files for uploading
+        shell: bash
+        run: |
+          # Prevents the parent dist/ directory from being included
+          mv dist/facetracker ${{ env.dist-path }}
+          cp -r models ${{ env.dist-path }}
+
+      - name: Upload artifact
+        uses: ./.github/actions/upload-dated-artifact
+        with:
+          name: windows
+          path: dist # Add the parent directory to prevent a tarbomb
+
+  # Files must be packed into a tar in order for file permissions to be preserved
+  build-unix:
+    env:
+      artifact-path: OpenSeeFace
+      tar-path: OpenSeeFace.tar
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: linux
+            os: ubuntu-latest
+          - name: mac
+            os: macos-latest
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Pull OpenSeeFace
+        uses: actions/checkout@v3
+
+      - name: Create binary
+        shell: bash
+        run: |
+          chmod +x make_exe.sh
+          ./make_exe.sh
+
+      - name: Move files for uploading
+        shell: bash
+        run: |
+          # Prevents the parent dist/ directory from being included
+          mv dist/facetracker ${{ env.artifact-path }}
+          cp -r models ${{ env.artifact-path }}
+
+      - name: Fix permissions from PyInstaller
+        shell: bash
+        run: |
+          find ${{ env.artifact-path }} -type f -exec chmod -x {} +
+          chmod +x ${{ env.artifact-path }}/facetracker
+
+      - name: Pack files into tar
+        shell: bash
+        run: tar -cvf ${{ env.tar-path }} ${{ env.artifact-path }}
+
+      - name: Upload artifact
+        uses: ./.github/actions/upload-dated-artifact
+        with:
+          name: ${{ matrix.name }}
+          path: ${{ env.tar-path }}

Examples/Animations/JawOpen.anim.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 697b139e69dcb414498dcd93210620ff
+NativeFormatImporter:
+  externalObjects: {}
+  mainObjectFileID: 0
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Examples/OpenSeeWebcamInfo.cs

@@ -0,0 +1,270 @@
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Net;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+
+namespace OpenSee {
+
+[Serializable]
+public enum OpenSeeWebcamType { Unknown = -1, DirectShow, Blackmagic };
+
+[Serializable]
+public enum OpenSeeWebcamFormat { Any = 0, Unknown = 1, ARGB = 100, XRGB, I420 = 200, NV12, YV12, Y800, YVYU = 300, YUY2, UYVY, HDYC, MJPEG = 400, H264 };
+
+[Serializable]
+public struct OpenSeeWebcamCapability {
+    public int id;
+    
+    // DirectShow
+    public int minCX;
+    public int minCY;
+    public int maxCX;
+    public int maxCY;
+    public int granularityCX;
+    public int granularityCY;
+    public int minInterval;
+    public int maxInterval;
+    public int rating;
+    public OpenSeeWebcamFormat format;
+    
+    // Blackmagic
+    public int bmTimescale;
+    public int bmFrameduration;
+    public int bmModecode;
+}
+
+[Serializable]
+public class OpenSeeWebcam {
+    public OpenSeeWebcamType type = OpenSeeWebcamType.Unknown;
+    public int id;
+    public string name;
+
+    // DirectShow
+    public string path;
+    
+    // Blackmagic
+    public long bmId;
+    public int bmFlags;
+
+    // General
+    public OpenSeeWebcamCapability[] caps;
+    
+    public List<string> prettyCaps = null;
+    
+    private List<Tuple<OpenSeeWebcamCapability, int>> splitCaps = null;
+    
+    bool ge(bool a, bool b) {
+        return a || (!a && !b);
+    }
+
+    private int CompareCaps(OpenSeeWebcamCapability a, OpenSeeWebcamCapability b) {
+        int aRes = a.minCX * a.minCY;
+        int bRes = b.minCX * b.minCY;
+        bool aGoodFps = a.minInterval <= 670000;
+        bool bGoodFps = a.minInterval <= 670000;
+        bool aGoodRes = aRes >= 850000 && aRes <= 2200000;
+        bool bGoodRes = bRes >= 850000 && bRes <= 2200000;
+        if (a.rating < b.rating && ge(aGoodFps, bGoodFps) && ge(aGoodRes, bGoodRes))
+            return -1;
+        if (a.rating > b.rating && ge(bGoodFps, aGoodFps) && ge(bGoodRes, aGoodRes))
+            return 1;
+        if (aRes > bRes && ge(aGoodFps, bGoodFps) && ge(aGoodRes, bGoodRes))
+            return -1;
+        if (aRes < bRes && ge(bGoodFps, aGoodFps) && ge(bGoodRes, aGoodRes))
+            return 1;
+        if (a.minInterval < b.minInterval)
+            return -1;
+        if (a.minInterval > b.minInterval)
+            return 1;
+        return 0;
+    }
+    
+    private string GetPrettyCapability(OpenSeeWebcamCapability cap) {
+        float fps = 10000000f / (float)cap.minInterval;
+        string prettyCap = cap.minCX + "x" + cap.minCY + ", " + fps.ToString("0.##") + "fps (" + cap.format.ToString() + ")";
+        return prettyCap;
+    }
+    
+    public List<string> GetPrettyCapabilities() {
+        if (splitCaps != null && prettyCaps != null)
+            return prettyCaps;
+        
+        splitCaps = new List<Tuple<OpenSeeWebcamCapability, int>>();
+        
+        for (int i = 0; i < caps.Length; i++) {
+            if (caps[i].minCX == caps[i].maxCX && caps[i].minCY == caps[i].maxCY)
+                splitCaps.Add(new Tuple<OpenSeeWebcamCapability, int>(caps[i], i));
+            else {
+                OpenSeeWebcamCapability min = caps[i];
+                OpenSeeWebcamCapability max = caps[i];
+                min.maxCX = min.minCX;
+                min.maxCY = min.minCY;
+                max.minCX = max.maxCX;
+                max.minCY = max.maxCY;
+                splitCaps.Add(new Tuple<OpenSeeWebcamCapability, int>(min, i));
+                splitCaps.Add(new Tuple<OpenSeeWebcamCapability, int>(max, i));
+            }
+        }
+        
+        splitCaps.Sort((a, b) => CompareCaps(a.Item1, b.Item1));
+        
+        prettyCaps = new List<string>();
+        if (type == OpenSeeWebcamType.DirectShow)
+            prettyCaps.Add("Default settings");
+        foreach (var cap in splitCaps) {
+            prettyCaps.Add(GetPrettyCapability(cap.Item1));
+        }
+        
+        return prettyCaps;
+    }
+    
+    public OpenSeeWebcamCapability GetCapabilityByPrettyIndex(int index) {
+        int dshowOffset = 0;
+        if (type == OpenSeeWebcamType.DirectShow) {
+            if (index == 0) {
+                OpenSeeWebcamCapability cap = new OpenSeeWebcamCapability();
+                cap.id = -1;
+                cap.format = OpenSeeWebcamFormat.Any;
+                cap.minInterval = 10000000 / 60;
+                return cap;
+            }
+            dshowOffset = 1;
+        }
+        
+        if (splitCaps == null || index >= splitCaps.Count + dshowOffset || index < 0)
+            throw new InvalidOperationException("Invalid capability index.");
+        
+        return caps[splitCaps[index - dshowOffset].Item2];
+    }
+}
+
+[DefaultExecutionOrder(-50)]
+public class OpenSeeWebcamInfo : MonoBehaviour {
+    #region DllImport
+    // DirectShow
+    [DllImport("dshowcapture_x64", CallingConvention = CallingConvention.Cdecl, EntryPoint = "create_capture")]
+	private static extern System.IntPtr create_capture_x64();
+
+    [DllImport("dshowcapture_x64", CallingConvention = CallingConvention.Cdecl, EntryPoint = "get_json_length")]
+	private static extern int get_json_length_x64(System.IntPtr cap);
+
+    [DllImport("dshowcapture_x64", CallingConvention = CallingConvention.Cdecl, EntryPoint = "get_json")]
+	private static extern void get_json_x64(System.IntPtr cap, [Out] StringBuilder namebuffer, int bufferlength);
+    
+    [DllImport("dshowcapture_x64", CallingConvention = CallingConvention.Cdecl, EntryPoint = "destroy_capture")]
+	private static extern void destroy_capture_x64(System.IntPtr cap);
+
+    [DllImport("dshowcapture_x86", CallingConvention = CallingConvention.Cdecl, EntryPoint = "create_capture")]
+	private static extern System.IntPtr create_capture_x86();
+
+    [DllImport("dshowcapture_x86", CallingConvention = CallingConvention.Cdecl, EntryPoint = "get_json_length")]
+	private static extern int get_json_length_x86(System.IntPtr cap);
+
+    [DllImport("dshowcapture_x86", CallingConvention = CallingConvention.Cdecl, EntryPoint = "get_json")]
+	private static extern void get_json_x86(System.IntPtr cap, [Out] StringBuilder namebuffer, int bufferlength);
+
+    [DllImport("dshowcapture_x86", CallingConvention = CallingConvention.Cdecl, EntryPoint = "destroy_capture")]
+	private static extern void destroy_capture_x86(System.IntPtr cap);
+
+    // Blackmagic
+    [DllImport("libminibmcapture64", CallingConvention = CallingConvention.Cdecl, EntryPoint = "get_json_length")]
+	private static extern int bm_get_json_length_x64();
+
+    [DllImport("libminibmcapture64", CallingConvention = CallingConvention.Cdecl, EntryPoint = "get_json")]
+	private static extern void bm_get_json_x64([Out] StringBuilder namebuffer, int bufferlength);
+
+    [DllImport("libminibmcapture32", CallingConvention = CallingConvention.Cdecl, EntryPoint = "get_json_length")]
+	private static extern int bm_get_json_length_x86();
+
+    [DllImport("libminibmcapture32", CallingConvention = CallingConvention.Cdecl, EntryPoint = "get_json")]
+	private static extern void bm_get_json_x86([Out] StringBuilder namebuffer, int bufferlength);
+    #endregion
+    
+    [Serializable]
+    public class OpenSeeWebcamList {
+        public List<OpenSeeWebcam> list;
+    }
+
+    public List<OpenSeeWebcam> cameras;
+    public bool dumpJson = false;
+    
+    private static bool dumpJsonStatic = false;
+    
+    static private string ListCameraDetails_x64() {
+        System.IntPtr cap = create_capture_x64();
+		int length = get_json_length_x64(cap);
+        StringBuilder buffer = new StringBuilder(length);
+        get_json_x64(cap, buffer, length);
+        destroy_capture_x64(cap);
+		return buffer.ToString();
+    }
+
+    static private string ListCameraDetails_x86() {
+        System.IntPtr cap = create_capture_x86();
+		int length = get_json_length_x86(cap);
+        StringBuilder buffer = new StringBuilder(length);
+        get_json_x86(cap, buffer, length);
+        destroy_capture_x86(cap);
+		return buffer.ToString();
+    }
+
+    static private string ListBlackMagicDetails_x64() {
+		int length = bm_get_json_length_x64();
+        StringBuilder buffer = new StringBuilder(length);
+        bm_get_json_x64(buffer, length);
+		return buffer.ToString();
+    }
+
+    static private string ListBlackMagicDetails_x86() {
+		int length = bm_get_json_length_x86();
+        StringBuilder buffer = new StringBuilder(length);
+        bm_get_json_x86(buffer, length);
+		return buffer.ToString();
+    }
+
+    static public List<OpenSeeWebcam> ListCameraDetails(bool includeBlackMagic) {
+        string jsonData = null;
+        string bmJsonData = null;
+        if (Environment.Is64BitProcess) {
+            jsonData = ListCameraDetails_x64();
+            if (includeBlackMagic)
+                bmJsonData = ListBlackMagicDetails_x64();
+        } else {
+            jsonData = ListCameraDetails_x86();
+            if (includeBlackMagic)
+                bmJsonData = ListBlackMagicDetails_x86();
+        }
+        if (dumpJsonStatic) {
+            UnityEngine.Debug.Log("Camera JSON: " + jsonData);
+            if (includeBlackMagic)
+                UnityEngine.Debug.Log("Blackmagic JSON: " + bmJsonData);
+        }
+        List<OpenSeeWebcam> details = JsonUtility.FromJson<OpenSeeWebcamList>("{\"list\":" + jsonData + "}").list;
+        foreach (var cam in details)
+            cam.type = OpenSeeWebcamType.DirectShow;
+        if (includeBlackMagic) {
+            List<OpenSeeWebcam> bmDetails = JsonUtility.FromJson<OpenSeeWebcamList>("{\"list\":" + bmJsonData + "}").list;
+            foreach (var cam in bmDetails)
+                cam.type = OpenSeeWebcamType.Blackmagic;
+            details.AddRange(bmDetails);
+        }
+        return details;
+    }
+    
+    void Start() {
+        Initialize(false);
+    }
+    
+    public void Initialize(bool includeBlackMagic) {
+        dumpJsonStatic = dumpJson;
+        cameras = ListCameraDetails(includeBlackMagic);
+        foreach (var camera in cameras)
+            camera.GetPrettyCapabilities();
+    }
+}
+}

Examples/OpenSeeWebcamInfo.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 2e9598f1218283b418c7e72604be0335
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: -50
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Licenses/libminibmcapture.txt

@@ -0,0 +1,19 @@
+zlib License
+
+libminibmcapture (C) 2020 noorus
+
+This software is provided 'as-is', without any express or implied
+warranty.  In no event will the authors be held liable for any damages
+arising from the use of this software.
+
+Permission is granted to anyone to use this software for any purpose,
+including commercial applications, and to alter it and redistribute it
+freely, subject to the following restrictions:
+
+1. The origin of this software must not be misrepresented; you must not
+   claim that you wrote the original software. If you use this software
+   in a product, an acknowledgment in the product documentation would be
+   appreciated but is not required.
+2. Altered source versions must be plainly marked as such, and must not be
+   misrepresented as being the original software.
+3. This notice may not be removed or altered from any source distribution.

README.md

@@ -1,6 +1,8 @@
+![OSF.png](https://raw.githubusercontent.com/emilianavt/OpenSeeFace/master/Images/OSF.png)
+
 # Overview
 
-**Note**: This is a tracking library, **not** a stand-alone avatar puppeteering program. I'm also working on [such](https://www.vseeface.icu/) an application, which is currently in a semi-public beta, with releases being posted in the `#vseeface` channel of [@Virtual_Deat](https://twitter.com/Virtual_Deat)'s Discord server.
+**Note**: This is a tracking library, **not** a stand-alone avatar puppeteering program. I'm also working on [VSeeFace](https://www.vseeface.icu/), which allows animating [VRM](https://vrm.dev/en/how_to_make_vrm/) and [VSFAvatar](https://www.youtube.com/watch?v=jhQ8DF87I5I) 3D models by using OpenSeeFace tracking. [VTube Studio](https://denchisoft.com/) uses OpenSeeFace for webcam based tracking to animate Live2D models. A renderer for the Godot engine can be found [here](https://github.com/virtual-puppet-project/vpuppr).
 
 This project implements a facial landmark detection model based on MobileNetV3.
 
@@ -8,14 +10,20 @@ As Pytorch 1.3 CPU inference speed on Windows is very low, the model was convert
 
 If anyone is curious, the name is a silly pun on the open seas and seeing faces. There's no deeper meaning.
 
-[Unity sample video](https://twitter.com/emiliana_vt/status/1210622149314203648) | [Sample video](https://www.youtube.com/watch?v=AOPHiAp9DBE) | [Sample video](https://www.youtube.com/watch?v=-cBSuHGdBWQ) | [Model comparison](https://www.youtube.com/watch?v=yMcJszUy7FA)
-
-These sample videos use older versions of the tracking models. Tracking accuracy of the current models is better.
-
 An up to date sample video can be found [here](https://www.youtube.com/watch?v=AaNap_ud_3I&vq=hd1080), showing the default tracking model's performance under different noise and light levels.
 
+# Tracking quality
+
+Since the landmarks used by OpenSeeFace are a bit different from those used by other approaches (they are close to iBUG 68, with two less points in the mouth corners and quasi-3D face contours instead of face contours that follow the visible outline) it is hard to numerically compare its accuracy to that of other approaches found commonly in scientific literature. The tracking performance is also more optimized for making landmarks that are useful for animating an avatar than for exactly fitting the face image. For example, as long as the eye landmarks show whether the eyes are opened or closed, even if their location is somewhat off, they can still be useful for this purpose.
+
+From general observation, OpenSeeFace performs well in adverse conditions (low light, high noise, low resolution) and keeps tracking faces through a very wide range of head poses with relatively high stability of landmark positions. Compared to MediaPipe, OpenSeeFace landmarks remain more stable in challenging conditions and it accurately represents a wider range of mouth poses. However, tracking of the eye region can be less accurate.
+
+I ran OpenSeeFace on a sample clip from the video presentation for [3D Face Reconstruction with Dense Landmarks](https://microsoft.github.io/DenseLandmarks/) by Wood et al. to compare it to MediaPipe and their approach. You can watch the result [here](https://www.vseeface.icu/assets/media/OSFMediaPipe3DFR.mp4).
+
 # Usage
 
+A sample Unity project for VRM based avatar animation can be found [here](https://github.com/emilianavt/OpenSeeFaceSample).
+
 The face tracking itself is done by the `facetracker.py` Python 3.7 script. It is a commandline program, so you should start it manually from cmd or write a batch file to start it. If you downloaded a release and are on Windows, you can run the `facetracker.exe` inside the `Binary` folder without having Python installed. You can also use the `run.bat` inside the `Binary` folder for a basic demonstration of the tracker.
 
 The script will perform the tracking on webcam input or video file and send the tracking data over UDP. This design also allows tracking to be done on a separate PC from the one who uses the tracking information. This can be useful to enhance performance and to avoid accidentially revealing camera footage.
@@ -30,6 +38,8 @@ A simple demonstration can be achieved by creating a new scene in Unity, adding
 
     python facetracker.py --visualize 3 --pnp-points 1 --max-threads 4 -c video.mp4
 
+__Note__: If dependencies were installed using [poetry](https://python-poetry.org/), the commands have to be executed from a `poetry shell` or have to be prefixed with `poetry run`.
+
 This way the tracking script will output its own tracking visualization while also demonstrating the transmission of tracking data to Unity.
 
 The included `OpenSeeLauncher` component allows starting the face tracker program from Unity. It is designed to work with the pyinstaller created executable distributed in the binary release bundles. It provides three public API functions:
@@ -91,7 +101,7 @@ Four pretrained face landmark models are included. Using the `--model` switch, i
 
 FPS measurements are from running on one core of my CPU.
 
-Pytorch weights for use with `model.py` can be found [here](https://mega.nz/file/vvYXlYQT#h7FpEg4tmOCJNxjpsDEw0JomJIkVGKwrt4OUV0RNDDU).
+Pytorch weights for use with `model.py` can be found [here](https://mega.nz/file/vvYXlYQT#h7FpEg4tmOCJNxjpsDEw0JomJIkVGKwrt4OUV0RNDDU). Some unoptimized ONNX models can be found [here](https://github.com/emilianavt/OpenSeeFace/issues/48).
 
 # Results
 
@@ -103,12 +113,6 @@ Pytorch weights for use with `model.py` can be found [here](https://mega.nz/file
 
 More samples: [Results3.png](https://raw.githubusercontent.com/emilianavt/OpenSeeFace/master/Images/Results3.png), [Results4.png](https://raw.githubusercontent.com/emilianavt/OpenSeeFace/master/Images/Results4.png)
 
-Here's an informal [comparison](https://raw.githubusercontent.com/emilianavt/OpenSeeFace/master/Images/3DDFA_V2.jpg) with [3DDFA_V2](https://github.com/cleardusk/3DDFA_V2) on five arbitrary, non-cherry picked examples from my test data:
-
-![3DDFA_V2_800.png](https://raw.githubusercontent.com/emilianavt/OpenSeeFace/master/Images/3DDFA_V2_800.jpg)
-
-Please ignore the blue eye points in the OpenSeeFace images as they are not actually part of the tracking results.
-
 ## Face detection
 
 The landmark model is quite robust with respect to the size and orientation of the faces, so the custom face detection model gets away with rougher bounding boxes than other approaches. It has a favorable speed to accuracy ratio for the purposes of this project.
@@ -125,9 +129,8 @@ When distributing it, you should also distribute the `Licenses` folder along wit
 
 The release builds contain a custom build of ONNX Runtime without telemetry.
 
-# Dependencies
+# Dependencies (Python 3.6 - 3.9)
 
-* Python 3.7
 * ONNX Runtime
 * OpenCV
 * Pillow
@@ -137,6 +140,22 @@ The required libraries can be installed using pip:
 
      pip install onnxruntime opencv-python pillow numpy
 
+Alternatively poetry can be used to 
+install all dependencies for this project in a separate virtual env:
+
+     poetry install
+
+# Dependencies
+
+* onnxruntime
+* OpenCV
+* Pillow
+* Numpy
+
+The required libraries can be installed using pip:
+ 
+	pip install onnxruntime opencv-python pillow numpy
+
 # References
 
 ## Training dataset
@@ -162,6 +181,8 @@ Additional training has been done on the WFLW dataset after reducing it to 66 po
 
 For the training the gaze and blink detection model, the [MPIIGaze](https://www.mpi-inf.mpg.de/departments/computer-vision-and-machine-learning/research/gaze-based-human-computer-interaction/appearance-based-gaze-estimation-in-the-wild/) dataset was used. Additionally, around 125000 synthetic eyes generated with [UnityEyes](https://www.cl.cam.ac.uk/research/rainbow/projects/unityeyes/) were used during training.
 
+It should be noted that additional custom data was also used during the training process and that the reference landmarks from the original datasets have been modified in certain ways to address various issues. It is likely not possible to reproduce these models with just the original LS3D-W and WFLW datasets, however the additional data is not redistributable.
+
 The heatmap regression based face detection model was trained on random 224x224 crops from the WIDER FACE dataset.
 
 	@inproceedings{yang2016wider,

Unity/OpenSee.cs

@@ -24,6 +24,8 @@ public class OpenSee : MonoBehaviour {
     [Tooltip("This contains the actual tracking data")]
     public OpenSeeData[] trackingData = null;
     
+    public bool listening { get; private set; } = false;
+    
     [HideInInspector]
     public float maxFit3DError = 100f;
 
@@ -39,9 +41,9 @@ public class OpenSee : MonoBehaviour {
         public float rightEyeOpen;
         [Tooltip("This field tells you how likely it is that the left eye is open.")]
         public float leftEyeOpen;
-        [Tooltip("This field tells you how likely it is that the right eye is open.")]
+        [Tooltip("This field contains the rotation of the right eyeball.")]
         public Quaternion rightGaze;
-        [Tooltip("This field tells you how likely it is that the left eye is open.")]
+        [Tooltip("This field contains the rotation of the left eyeball.")]
         public Quaternion leftGaze;
         [Tooltip("This field tells you if 3D points have been successfully estimated from the 2D points. If this is false, do not rely on pose or 3D data.")]
         public bool got3DPoints;
@@ -211,6 +213,7 @@ public class OpenSee : MonoBehaviour {
 
     void performReception() {
         EndPoint senderRemote = new IPEndPoint(IPAddress.Any, 0);
+        listening = true;
         while (!stopReception) {
             try {
                 int receivedBytes = socket.ReceiveFrom(buffer, SocketFlags.None, ref senderRemote);

Unity/OpenSeeExpression.cs

@@ -13,6 +13,8 @@ public class OpenSeeExpression : MonoBehaviour
     [Header("Settings")]
     [Tooltip("This is the source of face tracking data. If it is not set, any OpenSee component on the current object is used.")]
     public OpenSee openSee;
+    [Tooltip("When enabled ThunderSVM is used instead of LibSVM, which is usually faster, but may cause issues on some systems.")]
+    public bool useThunderSVM = false;
     [Header("Simple mode")]
     [Tooltip("When enabled, some simple heuristics will be used to detect expressions, rather than an SVM model. Outputs one of: neutral, fun, angry, surprise")]
     public bool simpleMode = false;
@@ -164,10 +166,10 @@ public class OpenSeeExpression : MonoBehaviour
                 oser = formatter.Deserialize(gzipStream) as OpenSeeExpressionRepresentation;
             }
             expressions = oser.expressions;
-            if (oser.thunderSVM)
+            /*if (oser.thunderSVM)
                 model = new ThunderSVMModel(oser.modelBytes);
-            else
+            else*/
-                model = new SVMModel(oser.modelBytes);
+            model = new SVMModel(oser.modelBytes);
             classLabels = oser.classLabels;
             indices = oser.indices;
             //pointSelection = oser.pointSelection;
@@ -189,7 +191,7 @@ public class OpenSeeExpression : MonoBehaviour
             oser.classLabels = classLabels;
             oser.indices = indices;
             oser.pointSelection = pointSelection;
-            oser.thunderSVM = true;
+            oser.thunderSVM = model is ThunderSVMModel;
             //oser.newModel = true;
 
             IFormatter formatter = new BinaryFormatter();
@@ -362,7 +364,10 @@ public class OpenSeeExpression : MonoBehaviour
         }
         ResetInfo();
         expressions = new Dictionary<string, List<float[]>>();
-        model = new ThunderSVMModel();
+        if (useThunderSVM)
+            model = new ThunderSVMModel();
+        else
+            model = new SVMModel();
         rnd = new System.Random();
     }
 
@@ -416,8 +421,10 @@ public class OpenSeeExpression : MonoBehaviour
             return false;
         }
         modelReady = false;
-        if (model is SVMModel)
+        if (model is SVMModel && useThunderSVM)
             model = new ThunderSVMModel();
+        else if (model is ThunderSVMModel && !useThunderSVM)
+            model = new SVMModel();
         SelectPoints();
         List<string> keys = new List<string>();
         List<string> accuracyWarnings = new List<string>();

Unity/OpenSeeIKTarget.cs

@@ -2,17 +2,24 @@
 
 namespace OpenSee {
 
+[DefaultExecutionOrder(-1)]
 public class OpenSeeIKTarget : MonoBehaviour
 {
     [Header("Settings")]
     [Tooltip("This is the source of face tracking data.")]
     public OpenSee openSee;
+    [Tooltip("When this is set to another game object, its kinematic interpolation component will get updated from values of this.")]
+    public OpenSeeKinematicInterpolation kinematicInterpolation;
+    [Tooltip("When set, the this object will follow the specified transform instead and automatically recalibrate based on it.")]
+    public Transform externalTarget = null;
     [Tooltip("This can be used to skip the first few tracking frames.")]
     public int skipFirst = 5;
     [Tooltip("When set to true, the next rotation and translation values will be set as zero rotation and zero translation.")]
     public bool calibrate = true;
     [Tooltip("This sets which face id to look for in the OpenSee data.")]
     public int faceId = 0;
+    [Tooltip("When enabled, the IK target will not move up and down.")]
+    public bool preventVerticalMotion = false;
     [Tooltip("When this is enabled, rotations and movement are mirrored.")]
     public bool mirrorMotion = false;
     [Tooltip("Often, the translation vector's scale is too high. Setting this somewhere between 0.05 to 0.3 seems stabilize things, but it also reduces the range of motion.")]
@@ -80,17 +87,46 @@ public class OpenSeeIKTarget : MonoBehaviour
     private bool skippedPosition = false;
     private float skipStartTimePosition = -1f;
     
+    private Transform lastExternalTarget = null;
+    
     public void Calibrate() {
         calibrate = true;
     }
+    
+    public static Vector3 PrettyEuler(Vector3 last, Vector3 current) {
+        // Ew, but how would you do componentwise filtering/interpolation with quaternions?
+        if (Mathf.Abs(current.x - last.x) > Mathf.Abs((current.x - 360f) - last.x))
+            current.x -= 360f;
+        else if (Mathf.Abs(current.x - last.x) > Mathf.Abs((current.x + 360f) - last.x))
+            current.x += 360f;
+        if (Mathf.Abs(current.y - last.y) > Mathf.Abs((current.y - 360f) - last.y))
+            current.y -= 360f;
+        else if (Mathf.Abs(current.y - last.y) > Mathf.Abs((current.y + 360f) - last.y))
+            current.x += 360f;
+        if (Mathf.Abs(current.z - last.z) > Mathf.Abs((current.z - 360f) - last.z))
+            current.z -= 360f;
+        else if (Mathf.Abs(current.z - last.z) > Mathf.Abs((current.z + 360f) - last.z))
+            current.z += 360f;
+        return current;
+    }
+    
+    Vector3 FilterPos(Vector3 a, Vector3 b) {
+        return Vector3.Lerp(a, b, 1f - smooth);
+    }
+    
+    Quaternion FilterRot(Quaternion a, Quaternion b) {
+        return Quaternion.Lerp(a, b, 1f - smooth);
+    }
 
     void Interpolate() {
         if (!interpolate || interpolateState < 2)
             return;
-        float t = Mathf.Clamp((float)interpolationCount / avgInterps, 0f, 0.985f);
+        float t = Mathf.Clamp((float)interpolationCount / avgInterps, 0f, 1.5f);
-        transform.localPosition = Vector3.Lerp(lastT, currentT, t);
+        transform.localPosition = Vector3.SlerpUnclamped(lastT, currentT, t);
-        transform.localRotation = Quaternion.Lerp(lastR, currentR, t);
+        transform.localRotation = Quaternion.SlerpUnclamped(lastR, currentR, t);
         interpolationCount++;
+        //transform.localPosition = new Vector3(0.3f * ((Time.time/3f) % 3f) - 0.5f, 0.3f * Mathf.Sin(Mathf.Rad2Deg * ((Time.time/5f) % 3f)), 0f);
+        //transform.localRotation = Quaternion.identity * Quaternion.AngleAxis(20f * Mathf.Sin(Mathf.Rad2Deg * ((Time.time/5f) % 3f)) - 10f, Vector3.up);
     }
 
     Quaternion MirrorQuaternion(Quaternion q) {
@@ -103,23 +139,33 @@ public class OpenSeeIKTarget : MonoBehaviour
 
     void RunUpdate() {
         var openSeeData = openSee.GetOpenSeeData(faceId);
-        if (openSeeData == null || openSeeData.fit3DError > openSee.maxFit3DError)
+        Quaternion convertedQuaternion;
-            return;
+        Vector3 t;
-        if (openSeeData.time > updated) {
+        Vector3 convertedTranslation;
-            updated = openSeeData.time;
+        
-            trackingFrames++;
+        if (externalTarget == null) {
-            if (trackingFrames < skipFirst)
+            if (openSeeData == null || openSeeData.fit3DError > openSee.maxFit3DError)
                 return;
+            if (openSeeData.time > updated) {
+                updated = openSeeData.time;
+                trackingFrames++;
+                if (trackingFrames < skipFirst)
+                    return;
+            } else {
+                Interpolate();
+                return;
+            }
+            
+            convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w);
+            t = openSeeData.translation;
+            t.x = -t.x;
+            t.z = -t.z;
+            convertedTranslation = new Vector3(t.x, t.y, t.z);
         } else {
-            Interpolate();
+            convertedQuaternion = externalTarget.localRotation;
-            return;
+            convertedTranslation = externalTarget.localPosition;
+            t = convertedTranslation;
         }
-
-        Quaternion convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w);
-        Vector3 t = openSeeData.translation;
-        t.x = -t.x;
-        t.z = -t.z;
-        Vector3 convertedTranslation = new Vector3(t.x, t.y, t.z);
         
         // Check for angular outliers
         if (gotAccepted) {
@@ -184,6 +230,9 @@ public class OpenSeeIKTarget : MonoBehaviour
             outlierSkipsDistance += 1;
             return;
         }
+        
+        calibrate = calibrate || (externalTarget != lastExternalTarget);
+        lastExternalTarget = externalTarget;
 
         if (calibrate) {
             dR = convertedQuaternion;
@@ -210,29 +259,43 @@ public class OpenSeeIKTarget : MonoBehaviour
             convertedQuaternion = MirrorQuaternion(convertedQuaternion);
             t = MirrorTranslation(t);
         }
-
+        
         if (interpolateState > 1)
             avgInterps = Mathf.Lerp(avgInterps, (float)interpolationCount, 0.15f);
         interpolationCount = 0;
         averageInterpolations = avgInterps;
-
+        
-        lastT = currentT;
+        lastT = transform.localPosition;
-        lastR = currentR;
+        lastR = transform.localRotation;
         if (interpolateState > 0) {
-            currentT = Vector3.Lerp(currentT, (t - dT) * translationScale, 1f - smooth);
+            currentT = FilterPos(transform.localPosition, (t - dT) * translationScale);
-            currentR = Quaternion.Lerp(transform.localRotation, convertedQuaternion * dR, 1f - smooth);
+            currentR = FilterRot(transform.localRotation, convertedQuaternion * dR);
         } else {
             currentT = (t - dT) * translationScale;
             currentR = convertedQuaternion * dR;
         }
         if (interpolateState < 2)
             interpolateState++;
-
+        
         if (interpolate) {
             Interpolate();
         } else {
-            transform.localPosition = Vector3.Lerp(transform.localPosition, (t - dT) * translationScale, 1f - smooth);
+            transform.localPosition = FilterPos(transform.localPosition, (t - dT) * translationScale);
-            transform.localRotation = Quaternion.Lerp(transform.localRotation, convertedQuaternion * dR, 1f - smooth);
+            transform.localRotation = FilterRot(transform.localRotation, convertedQuaternion * dR);
+        }
+
+       if (preventVerticalMotion) {
+           currentT.y = 0f;
+            Vector3 pos = transform.localPosition;
+            pos.y = 0f;
+            transform.localPosition = pos;
+        }
+
+        if (kinematicInterpolation != null && kinematicInterpolation.gameObject != gameObject) {
+            if (interpolate)
+                kinematicInterpolation.UpdateKI(updated, currentT, currentR);
+            else
+                kinematicInterpolation.UpdateKI(updated, transform.localPosition, transform.localRotation);
         }
         
         if (driftBack) {
@@ -247,6 +310,10 @@ public class OpenSeeIKTarget : MonoBehaviour
             translationOffset = new Vector3(dT.x, dT.y, dT.z);
        }
     }
+    
+    public void SetPreventVerticalMotion(bool v) {
+        preventVerticalMotion = v;
+    }
 
     void FixedUpdate()
     {

Unity/OpenSeeIKTarget.cs.meta

@@ -4,7 +4,7 @@ MonoImporter:
   externalObjects: {}
   serializedVersion: 2
   defaultReferences: []
-  executionOrder: 0
+  executionOrder: -5
   icon: {instanceID: 0}
   userData: 
   assetBundleName: 

Unity/OpenSeeInterpolation.cs

@@ -0,0 +1,232 @@
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+
+namespace OpenSee {
+    public class InterpolatedMap<T, V, U> where V : InterpolatedValue<U>, new() {
+        private Dictionary<T, V> map = new Dictionary<T, V>();
+        private float clamp = 1f;
+        private float smoothing = 0f;
+        private bool gotDefault = false;
+        private U defaultValue;
+
+        public void Clear() {
+            map.Clear();
+        }
+        
+        public void SetSmoothing(float smoothing) {
+            this.smoothing = smoothing;
+            foreach (var entry in map) {
+                if (entry.Value != null)
+                    entry.Value.SetSmoothing(smoothing);
+            }
+        }
+
+        public void SetClamp(float clamp) {
+            this.clamp = clamp;
+            foreach (var entry in map) {
+                if (entry.Value != null)
+                    entry.Value.SetClamp(clamp);
+            }
+        }
+        
+        void Prepare(T key) {
+            if (!map.ContainsKey(key)) {
+                map.Add(key, new V());
+                map[key].SetSmoothing(smoothing);
+                map[key].SetClamp(clamp);
+            }
+        }
+        
+        public void SetValue(T key, U v) {
+            Prepare(key);
+            map[key].SetValue(v);
+        }
+        
+        public void Store(T key, U v) {
+            Prepare(key);
+            map[key].UpdateValue(v);
+        }
+
+        public void Store(T key, U v, double nowT) {
+            if (!map.ContainsKey(key)) {
+                map.Add(key, new V());
+                map[key].SetSmoothing(smoothing);
+                map[key].SetClamp(clamp);
+            }
+            map[key].UpdateValue(v, nowT);
+        }
+        
+        public U GetDefault() {
+            if (gotDefault)
+                return defaultValue;
+            V tmp = new V();
+            defaultValue = tmp.GetDefault();
+            return defaultValue;
+        }
+        
+        public bool Check(T key) {
+            return map.ContainsKey(key);
+        }
+        
+        public U Get(T key) {
+            if (!map.ContainsKey(key))
+                return GetDefault();
+            return map[key].Interpolate();
+        }
+    }
+
+    abstract public class InterpolatedValue<V> {
+        private float interpolateT = 0f;
+        private float updateT = 0f;
+        private double lastT = 0f;
+        private double currentT = 0f;
+        private float clamp = 1f;
+        private float smoothing = 0f;
+        private int gotData = 0;
+
+        private bool haveDefault = false;
+        private V defaultValue;
+
+        private bool valueSet = false;
+        private V dataValue;
+        private V dataValueLast;
+
+        abstract public V ValueLerpUnclamped(V a, V b, float t);
+        abstract public V GetBuiltinDefault();
+        
+        public void SetDefault(V defaultValue) {
+            this.defaultValue = defaultValue;
+            haveDefault = true;
+        }
+        
+        public void ResetDefault() {
+            haveDefault = false;
+        }
+        
+        public V GetDefault() {
+            if (haveDefault)
+                return defaultValue;
+            return GetBuiltinDefault();
+        }
+        
+        public void SetSmoothing(float smoothing) {
+            this.smoothing = smoothing;
+        }
+
+        public void SetClamp(float clamp) {
+            this.clamp = clamp;
+        }
+        
+        public void SetValue(V v) {
+            dataValueLast = v;
+            dataValue = v;
+            valueSet = true;
+        }
+        
+        public InterpolatedValue(float smoothing = 0f, float clamp = 1f) {
+            this.smoothing = smoothing;
+            this.clamp = clamp;
+        }
+        
+        public void UpdateValue(V v) {
+            UpdateValue(v, Time.realtimeSinceStartup);
+        }
+        
+        public void UpdateValue(V v, double nowT) {
+            if (nowT < 0f)
+                return;
+            lock (this) {
+                if (nowT < currentT || gotData == 0) {
+                    // Reinitialize
+                    gotData = 1;
+                    updateT = Time.time;
+                    lastT = 0f;
+                    currentT = nowT;
+                    dataValueLast = GetDefault();
+                    dataValue = ValueLerpUnclamped(v, dataValueLast, smoothing);
+                } else if (nowT == currentT) {
+                    // Change value
+                    dataValue = ValueLerpUnclamped(v, dataValueLast, smoothing);
+                } else {
+                    // Regular operation
+                    updateT = Time.time;
+                    lastT = currentT;
+                    currentT = nowT;
+                    if (gotData < 2)
+                        gotData++;
+                    dataValueLast = dataValue;
+                    dataValue = ValueLerpUnclamped(v, dataValueLast, smoothing);
+                }
+            }
+        }
+        
+        public V Interpolate() {
+            lock (this) {
+                interpolateT = Time.time;
+                if (gotData < 2) {
+                    if (valueSet)
+                        return dataValue;
+                    else
+                        return GetDefault();
+                }
+                float t = Mathf.Min((interpolateT - updateT) / (float)(currentT - lastT), clamp);
+                return ValueLerpUnclamped(dataValueLast, dataValue, t);
+            }
+        }
+    }
+    
+    public class InterpolatedColor : InterpolatedValue<Color> {
+        public override Color ValueLerpUnclamped(Color a, Color b, float t) {
+            return Color.LerpUnclamped(a, b, t);
+        }
+        public override Color GetBuiltinDefault() {
+            return Color.clear;
+        }
+    }
+    
+    public class InterpolatedQuaternion : InterpolatedValue<Quaternion> {
+        public override Quaternion ValueLerpUnclamped(Quaternion a, Quaternion b, float t) {
+            return Quaternion.SlerpUnclamped(a, b, t);
+        }
+        public override Quaternion GetBuiltinDefault() {
+            return Quaternion.identity;
+        }
+    }
+
+    public class InterpolatedVector2 : InterpolatedValue<Vector2> {
+        public override Vector2 ValueLerpUnclamped(Vector2 a, Vector2 b, float t) {
+            return Vector2.LerpUnclamped(a, b, t);
+        }
+        public override Vector2 GetBuiltinDefault() {
+            return Vector2.zero;
+        }
+    }
+
+    public class InterpolatedVector3 : InterpolatedValue<Vector3> {
+        public override Vector3 ValueLerpUnclamped(Vector3 a, Vector3 b, float t) {
+            return Vector3.LerpUnclamped(a, b, t);
+        }
+        public override Vector3 GetBuiltinDefault() {
+            return Vector3.zero;
+        }
+    }
+
+    public class InterpolatedVector4 : InterpolatedValue<Vector4> {
+        public override Vector4 ValueLerpUnclamped(Vector4 a, Vector4 b, float t) {
+            return Vector4.LerpUnclamped(a, b, t);
+        }
+        public override Vector4 GetBuiltinDefault() {
+            return Vector4.zero;
+        }
+    }
+
+    public class InterpolatedFloat : InterpolatedValue<float> {
+        public override float ValueLerpUnclamped(float a, float b, float t) {
+            return Mathf.LerpUnclamped(a, b, t);
+        }
+        public override float GetBuiltinDefault() {
+            return 0f;
+        }
+    }
+}

Unity/OpenSeeInterpolation.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: d61e11f1b6900a7419383be1377f191b
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/OpenSeeKinematicInterpolation.cs

@@ -0,0 +1,108 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+
+namespace OpenSee {
+
+public class OpenSeeKinematicInterpolation : MonoBehaviour
+{
+    public float interpolateClamp = 1f;
+    
+    private Vector3 lastPos;
+    private Vector3 lastPosV;
+    private double lastT;
+    
+    private Vector3 currentPos;
+    private Vector3 currentPosV;
+    private double currentT = 0f;
+    
+    private Vector3 lastRot = Vector3.zero;
+    private Vector3 lastRotV;
+    
+    private Vector3 currentRot;
+    private Vector3 currentRotV;
+    
+    private float dT;
+    private Vector3 coeffPosA;
+    private Vector3 coeffPosB;
+    private Vector3 coeffRotA;
+    private Vector3 coeffRotB;
+    
+    private double updateT;
+    private double interpolateT;
+    private int ready = 0;
+    
+    public void UpdateKI(double newT, Vector3 newPos, Quaternion newRot) {
+        Interpolate();
+        if (newT - currentT > 3f)
+            ready = 0;
+
+        currentPos = transform.localPosition;
+        currentRot = OpenSeeIKTarget.PrettyEuler(currentRot, transform.localRotation.eulerAngles);
+        float dTNow = (float)(interpolateT - updateT);
+        if (interpolateT - updateT > 0 || ready < 1) {
+            currentPosV = (currentPos - lastPos) / dT;
+            currentRotV = (currentRot - lastRot) / dT;
+        } else {
+            currentPosV = Vector3.zero;
+            currentRotV = Vector3.zero;
+        }
+
+        lastPos = currentPos;
+        lastPosV = currentPosV;
+        lastRot = currentRot;
+        lastRotV = currentRotV;
+        lastT = currentT;
+        
+        currentPos = newPos;
+        currentRot = OpenSeeIKTarget.PrettyEuler(lastRot, newRot.eulerAngles);
+        currentT = newT;
+        dT = (float)(currentT - lastT);
+        currentPosV = (currentPos - lastPos) / dT;
+        currentRotV = (currentRot - lastRot) / dT;
+
+        float dTPow2 = Mathf.Pow(dT, 2);
+        float dTPow3 = Mathf.Pow(dT, 3);
+        coeffPosA = (- (2f*(currentPosV * dT + 2f * dT * lastPosV - 3f * currentPos + 3f * lastPos)) / dTPow2) / 2f;
+        coeffPosB = ((6f*(currentPosV * dT + dT * lastPosV - 2f * currentPos + 2f * lastPos)) / dTPow3)/6f;
+        coeffRotA = (- (2f*(currentRotV * dT + 2f * dT * lastRotV - 3f * currentRot + 3f * lastRot)) / dTPow2) / 2f;
+        coeffRotB = ((6f*(currentRotV * dT + dT * lastRotV - 2f * currentRot + 2f * lastRot)) / dTPow3)/6f;
+        
+        updateT = Time.time;
+        if (ready <= 2)
+            ready++;
+    }
+    
+    float InterpolateValue(float dTNow, float dTNowPow2, float dTNowPow3, float a, float b, float x1, float x2, float v1, float v2) {
+        return x1 + v1 * dTNow + dTNowPow2 * a + dTNowPow3 * b;
+    }
+    
+    void Interpolate() {
+        interpolateT = Time.time;
+        if (interpolateT == updateT || ready <= 2)
+            return;
+        float dTNow = (float)(interpolateT - updateT);
+        if (dTNow >= dT) {
+            transform.localPosition = Vector3.LerpUnclamped(lastPos, currentPos, Mathf.Min((float)(dTNow / dT), interpolateClamp));
+            transform.localEulerAngles = Vector3.LerpUnclamped(lastRot, currentRot, Mathf.Min((float)(dTNow / dT), interpolateClamp));
+        } else {
+            float dTNowPow2 = Mathf.Pow(dTNow, 2);
+            float dTNowPow3 = Mathf.Pow(dTNow, 3);
+            float x = InterpolateValue(dTNow, dTNowPow2, dTNowPow3, coeffPosA.x, coeffPosB.x, lastPos.x, currentPos.x, lastPosV.x, currentPosV.x);
+            float y = InterpolateValue(dTNow, dTNowPow2, dTNowPow3, coeffPosA.y, coeffPosB.y, lastPos.y, currentPos.y, lastPosV.y, currentPosV.y);
+            float z = InterpolateValue(dTNow, dTNowPow2, dTNowPow3, coeffPosA.z, coeffPosB.z, lastPos.z, currentPos.z, lastPosV.z, currentPosV.z);
+            transform.localPosition = new Vector3(x, y, z);
+            x = InterpolateValue(dTNow, dTNowPow2, dTNowPow3, coeffRotA.x, coeffRotB.x, lastRot.x, currentRot.x, lastRotV.x, currentRotV.x);
+            y = InterpolateValue(dTNow, dTNowPow2, dTNowPow3, coeffRotA.y, coeffRotB.y, lastRot.y, currentRot.y, lastRotV.y, currentRotV.y);
+            z = InterpolateValue(dTNow, dTNowPow2, dTNowPow3, coeffRotA.z, coeffRotB.z, lastRot.z, currentRot.z, lastRotV.z, currentRotV.z);
+            transform.localEulerAngles = new Vector3(x, y, z);
+        }
+    }
+    
+    public void Update() {
+        Interpolate();
+    }
+}
+
+}

Unity/OpenSeeKinematicInterpolation.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: a4112b4be90fc9e4baee9eade65378c3
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: -1
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/OpenSeeLauncher.cs

@@ -51,12 +51,14 @@ public class OpenSeeLauncher : MonoBehaviour {
     public bool autoListCameras = false;
     [Tooltip("If this is enabled, no output from the tracker will be printed to the debug console. This cannot be changed while the tracker is running.")]
     public bool dontPrint = true;
+    [Tooltip("If enabled, the next available port within 500 starting after the one given on the OpenSee component will be used.")]
+    public bool dynamicPort = false;
     [Tooltip("The path to \"facetracker.exe\".")]
     public string exePath = "facetracker.exe";
     [Tooltip("The path, where the .onnx model files can be found")]
     public string modelPath = "models\\";
     [Tooltip("Additional options that should be passed to the face tracker.")]
-    public List<string> commandlineOptions = new List<string>(new string[] { "--silent", "1", "--max-threads", "4" });
+    public List<string> commandlineOptions = new List<string>(new string[] { "--silent", "1", "--max-threads", "1" });
     [Tooltip("This string will be appended at the end of the tracker options without quoting.")]
     public string extraOptions = "";
     [Tooltip("If enabled, the tracker commandline option string will be logged with Debug.Log.")]
@@ -221,6 +223,48 @@ public class OpenSeeLauncher : MonoBehaviour {
         return cameras;
     }
     
+    public float[] Benchmark(int threads) {
+        if (!CheckSetup(false))
+            return null;
+
+        StringBuilder stringBuilder;
+        ProcessStartInfo processStartInfo;
+        Process process;
+
+        stringBuilder = new StringBuilder();
+
+        processStartInfo = new ProcessStartInfo();
+        processStartInfo.CreateNoWindow = true;
+        processStartInfo.RedirectStandardOutput = true;
+        processStartInfo.RedirectStandardInput = true;
+        processStartInfo.RedirectStandardError = true;
+        processStartInfo.UseShellExecute = false;
+        processStartInfo.FileName = exePath;
+        processStartInfo.Arguments = "--benchmark 1 --priority 4 --max-threads " + threads.ToString();
+        
+        process = new Process();
+        process.StartInfo = processStartInfo;
+        process.EnableRaisingEvents = true;
+        process.OutputDataReceived += new DataReceivedEventHandler(delegate(object sender, DataReceivedEventArgs e) {
+            stringBuilder.Append(e.Data);
+            stringBuilder.Append("\n");
+        });
+        process.Start();
+        job.AddProcess(process.Handle);
+        process.BeginOutputReadLine();
+        process.WaitForExit();
+        process.CancelOutputRead();
+        
+        string[] lines = stringBuilder.ToString().Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.RemoveEmptyEntries);
+        float[] results = new float[lines.Length];
+        
+        for (int i = 0; i < lines.Length; i++) {
+            results[i] = Single.Parse(lines[i]);
+        }
+        
+        return results;
+    }
+    
     // From: http://csharptest.net/529/how-to-correctly-escape-command-line-arguments-in-c/index.html
     private static string EscapeArguments(params string[] args)
     {
@@ -253,6 +297,27 @@ public class OpenSeeLauncher : MonoBehaviour {
         List<string> arguments = new List<string>();
         arguments.Add("--ip");
         arguments.Add(ip);
+        
+        if (dynamicPort && !openSeeTarget.listening) {
+            System.Net.IPEndPoint[] inUse = System.Net.NetworkInformation.IPGlobalProperties.GetIPGlobalProperties().GetActiveUdpListeners();
+            Array.Sort(inUse, delegate(System.Net.IPEndPoint a, System.Net.IPEndPoint b) { return a.Port.CompareTo(b.Port); });
+            int port = openSeeTarget.listenPort + 1;
+            for (int i = 0; i < inUse.Length; i++) {
+                if (inUse[i].Port < port)
+                    continue;
+                if (inUse[i].Port == port) {
+                    if (port <= openSeeTarget.listenPort + 500 && port <= IPEndPoint.MaxPort) {
+                        port++;
+                        continue;
+                    } else {
+                        return false;
+                    }
+                }
+                break;
+            }
+            openSeeTarget.listenPort = port;
+        }
+
         arguments.Add("--port");
         arguments.Add(openSeeTarget.listenPort.ToString());
         arguments.Add("--model-dir");
@@ -284,7 +349,7 @@ public class OpenSeeLauncher : MonoBehaviour {
             processStartInfo.RedirectStandardError = true;
             processStartInfo.UseShellExecute = false;
             processStartInfo.FileName = exePath;
-            processStartInfo.Arguments = "facetracker " + argumentString;
+            processStartInfo.Arguments = argumentString;
             
             trackerSB = new StringBuilder();
             trackerProcess = new Process();

Unity/OpenSeeShowPoints.cs

@@ -18,6 +18,7 @@ public class OpenSeeShowPoints : MonoBehaviour {
     public bool showLines = false;
     public float lineWidth = 0.01f;
     public Material lineMaterial;
+    public bool receiveShadows = false;
     
     private OpenSee.OpenSeeData openSeeData;
     private GameObject[] gameObjects;
@@ -46,6 +47,7 @@ public class OpenSeeShowPoints : MonoBehaviour {
             gameObjects[i].name = "Point " + (i + 1);
             if (material != null)
                 gameObjects[i].GetComponent<Renderer>().material = material;
+            gameObjects[i].GetComponent<Renderer>().receiveShadows = receiveShadows;
             gameObjects[i].layer = gameObject.layer;
             gameObjects[i].transform.SetParent(transform);
             gameObjects[i].transform.localScale = new Vector3(0.025f, 0.025f, 0.025f);
@@ -53,6 +55,7 @@ public class OpenSeeShowPoints : MonoBehaviour {
                 GameObject cylinder = GameObject.CreatePrimitive(PrimitiveType.Cylinder);
                 if (material != null)
                     cylinder.GetComponent<Renderer>().material = material;
+                cylinder.GetComponent<Renderer>().receiveShadows = receiveShadows;
                 cylinder.layer = gameObject.layer;
                 cylinder.transform.SetParent(gameObjects[i].transform);
                 cylinder.transform.localEulerAngles = new Vector3(90f, 0f, 0f);
@@ -85,6 +88,7 @@ public class OpenSeeShowPoints : MonoBehaviour {
                 lineRenderers[i].useWorldSpace = true;
                 lineRenderers[i].generateLightingData = true;
                 lineRenderers[i].material = lineMaterial;
+                lineRenderers[i].receiveShadows = receiveShadows;
                 lineRenderers[i].widthMultiplier = lineWidth;
                 lineRenderers[i].positionCount = 2;
         }
@@ -173,6 +177,7 @@ public class OpenSeeShowPoints : MonoBehaviour {
 
             for (int i = 0; i < 66; i++) {
                 Renderer renderer = gameObjects[i].GetComponent<Renderer>();
+                renderer.receiveShadows = receiveShadows;
                 renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i]));
                 float x = openSeeData.points[i].x;
                 float y = -openSeeData.points[i].y;
@@ -206,7 +211,7 @@ public class OpenSeeShowPoints : MonoBehaviour {
                 Color color = Color.Lerp(Color.red, Color.green, Mathf.Lerp(0.5f, openSeeData.confidence[a], openSeeData.confidence[b]));
                 lineRenderers[i].enabled = true;
                 lineRenderers[i].widthMultiplier = lineWidth;
-                //lineRenderers[i].material = lineMaterial;
+                lineRenderers[i].receiveShadows = receiveShadows;
                 lineRenderers[i].material.SetColor("_Color", color);
                 lineRenderers[i].startColor = color;
                 lineRenderers[i].endColor = color;

Unity/Plugins/dshowcapture_x64.dll.meta

@@ -0,0 +1,27 @@
+fileFormatVersion: 2
+guid: dd0c04eb3445c3048bfaa964c6bcb16f
+PluginImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  iconMap: {}
+  executionOrder: {}
+  defineConstraints: []
+  isPreloaded: 0
+  isOverridable: 0
+  isExplicitlyReferenced: 0
+  validateReferences: 1
+  platformData:
+  - first:
+      Any: 
+    second:
+      enabled: 1
+      settings: {}
+  - first:
+      Editor: Editor
+    second:
+      enabled: 0
+      settings:
+        DefaultValueInitialized: true
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/Plugins/dshowcapture_x86.dll.meta

@@ -0,0 +1,27 @@
+fileFormatVersion: 2
+guid: a504a0f747083b54c99d6e91731ccecc
+PluginImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  iconMap: {}
+  executionOrder: {}
+  defineConstraints: []
+  isPreloaded: 0
+  isOverridable: 0
+  isExplicitlyReferenced: 0
+  validateReferences: 1
+  platformData:
+  - first:
+      Any: 
+    second:
+      enabled: 1
+      settings: {}
+  - first:
+      Editor: Editor
+    second:
+      enabled: 0
+      settings:
+        DefaultValueInitialized: true
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/Plugins/libminibmcapture32.dll.meta

@@ -0,0 +1,27 @@
+fileFormatVersion: 2
+guid: 51dc8f0c65df30d4daa188cdfd1dd220
+PluginImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  iconMap: {}
+  executionOrder: {}
+  defineConstraints: []
+  isPreloaded: 0
+  isOverridable: 0
+  isExplicitlyReferenced: 0
+  validateReferences: 1
+  platformData:
+  - first:
+      Any: 
+    second:
+      enabled: 1
+      settings: {}
+  - first:
+      Editor: Editor
+    second:
+      enabled: 0
+      settings:
+        DefaultValueInitialized: true
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/Plugins/libminibmcapture64.dll.meta

@@ -0,0 +1,27 @@
+fileFormatVersion: 2
+guid: fcfc753ca92169e48aa69c1ecc1b0d64
+PluginImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  iconMap: {}
+  executionOrder: {}
+  defineConstraints: []
+  isPreloaded: 0
+  isOverridable: 0
+  isExplicitlyReferenced: 0
+  validateReferences: 1
+  platformData:
+  - first:
+      Any: 
+    second:
+      enabled: 1
+      settings: {}
+  - first:
+      Editor: Editor
+    second:
+      enabled: 0
+      settings:
+        DefaultValueInitialized: true
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

dshowcapture.py

@@ -5,53 +5,95 @@ import numpy as np
 from ctypes import *
 from PIL import Image
 import cv2
+import json
 
 def resolve(name):
     f = os.path.join(os.path.dirname(__file__), name)
     return f
 
 lib = None
+bm_lib = None
+bm_options = None
+bm_enabled = False
 
-def create_frame_buffer(width, height):
+def set_bm_enabled(v):
-    buffer = bytearray(width * height * 4)
+    global bm_enabled
+    bm_enabled = v
+
+def set_options(str):
+    global bm_options
+    bm_options = str
+
+def create_frame_buffer(width, height, factor):
+    buffer = bytearray(width * height * 4 * factor)
     char_array = c_char * len(buffer)
     return char_array.from_buffer(buffer)
 
 class DShowCapture():
     def __init__(self):
         global lib
-        if lib is None:
+        global bm_lib
+        global bm_options
+        global bm_enabled
+        if lib is None or bm_lib is None:
             if platform.architecture()[0] == '32bit':
                 dll_path = resolve(os.path.join("dshowcapture", "dshowcapture_x86.dll"))
                 lib = cdll.LoadLibrary(dll_path)
+                if bm_enabled:
+                    dll_path = resolve(os.path.join("dshowcapture", "libminibmcapture32.dll"))
+                    bm_lib = cdll.LoadLibrary(dll_path)
             else:
                 dll_path = resolve(os.path.join("dshowcapture", "dshowcapture_x64.dll"))
                 lib = cdll.LoadLibrary(dll_path)
+                if bm_enabled:
+                    dll_path = resolve(os.path.join("dshowcapture", "libminibmcapture64.dll"))
+                    bm_lib = cdll.LoadLibrary(dll_path)
+
+            # DirectShow
             lib.create_capture.restype = c_void_p
             lib.get_devices.argtypes = [c_void_p]
             lib.get_device.argtypes = [c_void_p, c_int, c_char_p, c_int]
             lib.capture_device.argtypes = [c_void_p, c_int, c_int, c_int, c_int]
+            lib.capture_device_by_dcap.argtypes = [c_void_p, c_int, c_int, c_int, c_int, c_longlong]
             lib.capture_device_default.argtypes = [c_void_p, c_int]
             lib.get_width.argtypes = [c_void_p]
             lib.get_height.argtypes = [c_void_p]
             lib.get_fps.argtypes = [c_void_p]
             lib.get_flipped.argtypes = [c_void_p]
             lib.get_colorspace.argtypes = [c_void_p]
+            lib.get_colorspace_internal.argtypes = [c_void_p]
             lib.capturing.argtypes = [c_void_p]
             lib.get_frame.argtypes = [c_void_p, c_int, c_char_p, c_int]
             lib.get_size.argtypes = [c_void_p]
             lib.stop_capture.argtypes = [c_void_p]
+            lib.get_json_length.argtypes = [c_void_p]
+            lib.get_json.argtypes = [c_void_p, c_char_p, c_int]
             lib.destroy_capture.argtypes = [c_void_p]
+
+            # Blackmagic
+            if bm_enabled:
+                bm_lib.start_capture_single.argtypes = [c_int, c_int, c_void_p]
+                bm_lib.read_frame_bgra32_blocking.argtypes = [c_char_p, c_int]
+                bm_lib.stop_capture_single.argtypes = []
+                bm_lib.get_json_length.argtypes = []
+                bm_lib.get_json.argtypes = [c_char_p, c_int]
+                if bm_options is not None:
+                    bm_lib.set_options.argtypes = [c_char_p]
+                    bm_lib.set_options(bm_options.encode())
         self.lib = lib
+        if bm_enabled:
+            self.bm_lib = bm_lib
         self.cap = lib.create_capture()
         self.name_buffer = create_string_buffer(255);
         self.buffer = None
         self.have_devices = False
         self.size = None
         self.real_size = None
+        self.info = None
+        self.type = None
 
     def __del__(self):
-        if not self.buffer is None:
+        if self.buffer is not None:
             del self.buffer
         del self.name_buffer
         self.destroy_capture()
@@ -62,20 +104,88 @@ class DShowCapture():
 
     def get_device(self, device_number):
         self.lib.get_device(self.cap, device_number, self.name_buffer, 255)
-        name_str = str(self.name_buffer.value.decode('utf8'))
+        name_str = str(self.name_buffer.value.decode('utf8', 'surrogateescape'))
         return name_str
 
+    def get_info(self):
+        global bm_enabled
+        # DirectShow
+        json_length = self.lib.get_json_length(self.cap);
+        json_buffer = create_string_buffer(json_length)
+        self.lib.get_json(self.cap, json_buffer, json_length);
+        json_text = str(json_buffer.value.decode('utf8', 'surrogateescape'))
+        self.info = json.loads(json_text)
+        for cam in self.info:
+            cam["type"] = "DirectShow"
+            cam["index"] = cam["id"]
+        
+        # Blackmagic
+        if bm_enabled:
+            json_length = self.bm_lib.get_json_length();
+            json_buffer = create_string_buffer(json_length)
+            self.bm_lib.get_json(json_buffer, json_length);
+            json_text = str(json_buffer.value.decode('utf8', 'surrogateescape'))
+            bm_info = json.loads(json_text)
+            dshow_len = len(self.info)
+            for cam in bm_info:
+                cam["type"] = "Blackmagic"
+                cam["index"] = cam["id"] + dshow_len
+            self.info.extend(bm_info)
+
+        return self.info
+
     def capture_device(self, cam, width, height, fps):
         if not self.have_devices:
             self.get_devices()
         ret = self.lib.capture_device(self.cap, cam, width, height, fps) == 1
         if ret:
+            self.type = "DirectShow"
             self.width = self.get_width()
             self.height = self.get_height()
+            self.fps = self.get_fps()
             self.flipped = self.get_flipped()
             self.colorspace = self.get_colorspace()
+            self.colorspace_internal = self.get_colorspace_internal()
             self.size = self.width * self.height * 4
-            self.buffer = create_frame_buffer(self.width, self.height)
+            self.buffer = create_frame_buffer(self.width, self.height, 4)
+        else:
+            self.size = None
+            self.real_size = None
+        return ret;
+
+    def capture_device_by_dcap(self, cam, dcap, width, height, fps):
+        global bm_enabled
+        if not self.have_devices:
+            self.get_devices()
+        if self.info is None:
+            self.get_info()
+        fps = int(10000000 / fps)
+        ret = False
+        if self.info[cam]['type'] == "DirectShow":
+            ret = self.lib.capture_device_by_dcap(self.cap, cam, dcap, width, height, fps) == 1
+        elif bm_enabled and self.info[cam]['type'] == "Blackmagic":
+            ret = self.bm_lib.start_capture_single(self.info[cam]['id'], self.info[cam]['caps'][dcap]['bmModecode'], None) == 1
+        if ret:
+            self.type = self.info[cam]['type']
+            if self.type == "DirectShow":
+                self.width = self.get_width()
+                self.height = self.get_height()
+                self.fps = self.get_fps()
+                self.flipped = self.get_flipped()
+                self.colorspace = self.get_colorspace()
+                self.colorspace_internal = self.get_colorspace_internal()
+                self.size = self.width * self.height * 4 * 4
+                self.buffer = create_frame_buffer(self.width, self.height, 4)
+            elif bm_enabled and self.type == "Blackmagic":
+                self.width = self.info[cam]['caps'][dcap]['minCX']
+                self.height = self.info[cam]['caps'][dcap]['minCY']
+                self.fps = int(10000000 / self.info[cam]['caps'][dcap]['minInterval'])
+                self.flipped = True
+                self.colorspace = 101
+                self.colorspace_internal = self.colorspace
+                self.size = self.width * self.height * 4
+                self.buffer = create_frame_buffer(self.width, self.height, 1)
+                self.real_size = self.size
         else:
             self.size = None
             self.real_size = None
@@ -86,12 +196,15 @@ class DShowCapture():
             self.get_devices()
         ret = self.lib.capture_device_default(self.cap, cam) == 1
         if ret:
+            self.type = "DirectShow"
             self.width = self.get_width()
             self.height = self.get_height()
+            self.fps = self.get_fps()
             self.flipped = self.get_flipped()
             self.colorspace = self.get_colorspace()
-            self.size = self.width * self.height * 4
+            self.colorspace_internal = self.get_colorspace_internal()
-            self.buffer = create_frame_buffer(self.width, self.height)
+            self.size = self.width * self.height * 4 * 4
+            self.buffer = create_frame_buffer(self.width, self.height, 4)
         else:
             self.size = None
             self.real_size = None
@@ -110,52 +223,78 @@ class DShowCapture():
         return self.lib.get_flipped(self.cap) != 0
 
     def get_colorspace(self):
-        return self.lib.get_colorspace(self.cap)
+        colorspace = self.lib.get_colorspace(self.cap)
+        if colorspace == 0:
+            colorspace = self.lib.get_colorspace_internal(self.cap)
+        return colorspace
+
+    def get_colorspace_internal(self):
+        return self.lib.get_colorspace_internal(self.cap)
 
     def capturing(self):
+        if self.type == "Blackmagic":
+            return 1
         return self.lib.capturing(self.cap) == 1
 
     def get_frame(self, timeout):
+        global bm_enabled
         if self.size is None:
             return None
-        if self.lib.get_frame(self.cap, timeout, self.buffer, self.size) == 0:
+        if self.type == "DirectShow":
+            self.real_size = self.lib.get_frame(self.cap, timeout, self.buffer, self.size)
+        elif bm_enabled and self.type == "Blackmagic":
+            self.bm_lib.read_frame_bgra32_blocking(self.buffer, self.size)
+        else:
             return None
-        if self.real_size is None:
-            self.real_size = self.lib.get_size(self.cap)
         img = np.frombuffer(self.buffer, dtype=np.uint8)[0:self.real_size]
         if self.colorspace in [100, 101]:
-            img = cv2.cvtColor(img.reshape((self.height,self.width,4)), cv2.COLOR_BGRA2BGR);
+            if self.real_size == self.height * self.width * 4:
+                img = cv2.cvtColor(img.reshape((self.height,self.width,4)), cv2.COLOR_BGRA2BGR)
+            elif self.real_size == self.height * self.width * 3:
+                img = img.reshape((self.height,self.width,3))
+            else:
+                return None
             if not self.flipped:
                 img = cv2.flip(img, 0)
             return img
-        elif self.colorspace == 200:
+        elif self.colorspace == 200 and self.real_size == (3 * self.height // 2) * self.width:
-            img = cv2.cvtColor(img.reshape((3 * self.height // 2,self.width,1)), cv2.COLOR_YUV2BGR_I420);
+            img = cv2.cvtColor(img.reshape((3 * self.height // 2,self.width,1)), cv2.COLOR_YUV2BGR_I420)
-        elif self.colorspace == 201:
+        elif self.colorspace == 201 and self.real_size == (3 * self.height // 2) * self.width:
-            img = cv2.cvtColor(img.reshape((3 * self.height // 2,self.width,1)), cv2.COLOR_YUV2BGR_NV12);
+            img = cv2.cvtColor(img.reshape((3 * self.height // 2,self.width,1)), cv2.COLOR_YUV2BGR_NV12)
-        elif self.colorspace == 202:
+        elif self.colorspace == 202 and self.real_size == (3 * self.height // 2) * self.width:
-            img = cv2.cvtColor(img.reshape((3 * self.height // 2,self.width,1)), cv2.COLOR_YUV2BGR_YV12);
+            img = cv2.cvtColor(img.reshape((3 * self.height // 2,self.width,1)), cv2.COLOR_YUV2BGR_YV12)
-        elif self.colorspace == 203:
+        elif self.colorspace == 203 and self.real_size == self.height * self.width:
-            img = cv2.cvtColor(img.reshape((self.height,self.width,1)), cv2.COLOR_GRAY2BGR);
+            img = cv2.cvtColor(img.reshape((self.height,self.width,1)), cv2.COLOR_GRAY2BGR)
-        elif self.colorspace == 300:
+        elif self.colorspace == 300 and self.real_size == self.height * self.width * 2:
-            img = cv2.cvtColor(img.reshape((self.height,self.width,2)), cv2.COLOR_YUV2BGR_YVYU);
+            img = cv2.cvtColor(img.reshape((self.height,self.width,2)), cv2.COLOR_YUV2BGR_YVYU)
-        elif self.colorspace == 301:
+        elif self.colorspace == 301 and self.real_size == self.height * self.width * 2:
-            img = cv2.cvtColor(img.reshape((self.height,self.width,2)), cv2.COLOR_YUV2BGR_YUY2);
+            img = cv2.cvtColor(img.reshape((self.height,self.width,2)), cv2.COLOR_YUV2BGR_YUY2)
-        elif self.colorspace == 302:
+        elif self.colorspace == 302 and self.real_size == self.height * self.width * 2:
-            img = cv2.cvtColor(img.reshape((self.height,self.width,2)), cv2.COLOR_YUV2BGR_UYVY);
+            img = cv2.cvtColor(img.reshape((self.height,self.width,2)), cv2.COLOR_YUV2BGR_UYVY)
         elif self.colorspace == 303:
             return None
+        elif self.colorspace == 400 and self.real_size > 4:
+            img = cv2.imdecode(img, cv2.IMREAD_COLOR);
+        else:
+            return None
         if self.flipped:
             img = cv2.flip(img, 0)
         return img
 
     def stop_capture(self):
+        global bm_enabled
         self.size = None
         self.real_size = None
-        return self.lib.stop_capture(self.cap)
+        if self.type == "DirectShow":
+            return self.lib.stop_capture(self.cap)
+        elif bm_enabled and self.type == "Blackmagic":
+            return self.bm_lib.stop_capture_single() != 0
+        return True
 
     def destroy_capture(self):
         if self.cap is None:
             return 0
+        self.stop_capture()
         ret = self.lib.destroy_capture(self.cap)
         self.cap = None
         self.size = None
@@ -186,11 +325,11 @@ if __name__ == "__main__":
     width = cap.get_width()
     height = cap.get_height()
     flipped = cap.get_flipped()
-    print(f"Width: {width} Height: {height} FPS: {cap.get_fps()} Flipped: {flipped} Colorspace: {cap.get_colorspace()}")
+    print(f"Width: {width} Height: {height} FPS: {cap.get_fps()} Flipped: {flipped} Colorspace: {cap.get_colorspace()} Internal: {cap.get_colorspace_internal()}")
     
     while True:
         img = cap.get_frame(1000)
-        if not img is None:
+        if img is not None:
             cv2.imshow("DShowCapture", img)
             if cv2.waitKey(1) & 0xFF == ord('q'):
                 sys.exit(0)

facetracker.py

@@ -9,14 +9,20 @@ parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFo
 parser.add_argument("-i", "--ip", help="Set IP address for sending tracking data", default="127.0.0.1")
 parser.add_argument("-p", "--port", type=int, help="Set port for sending tracking data", default=11573)
 if os.name == 'nt':
-    parser.add_argument("-l", "--list-cameras", type=int, help="Set this to 1 to list the available cameras and quit, set this to a number higher than 1 to output only the names", default=0)
+    parser.add_argument("-l", "--list-cameras", type=int, help="Set this to 1 to list the available cameras and quit, set this to 2 or higher to output only the names", default=0)
+    parser.add_argument("-a", "--list-dcaps", type=int, help="Set this to -1 to list all cameras and their available capabilities, set this to a camera id to list that camera's capabilities", default=None)
     parser.add_argument("-W", "--width", type=int, help="Set camera and raw RGB width", default=640)
     parser.add_argument("-H", "--height", type=int, help="Set camera and raw RGB height", default=360)
-    parser.add_argument("-F", "--fps", type=int, help="Set camera frames per second", default=24)
+    parser.add_argument("-D", "--dcap", type=int, help="Set which device capability line to use or -1 to use the default camera settings (FPS still need to be set separately)", default=None)
+    parser.add_argument("-B", "--blackmagic", type=int, help="When set to 1, special support for Blackmagic devices is enabled", default=0)
 else:
     parser.add_argument("-W", "--width", type=int, help="Set raw RGB width", default=640)
     parser.add_argument("-H", "--height", type=int, help="Set raw RGB height", default=360)
+if sys.platform == 'linux':
+    parser.add_argument("--dformat", type=str, help="Set which device format to use (MJPG, YUYV, RGB3, ...)", default=None)
+parser.add_argument("-F", "--fps", type=int, help="Set camera frames per second", default=24)
 parser.add_argument("-c", "--capture", help="Set camera ID (0, 1...) or video file", default="0")
+parser.add_argument("-M", "--mirror-input", action="store_true", help="Process a mirror image of the input video")
 parser.add_argument("-m", "--max-threads", type=int, help="Set the maximum number of threads", default=1)
 parser.add_argument("-t", "--threshold", type=float, help="Set minimum confidence threshold for face tracking", default=None)
 parser.add_argument("-d", "--detection-threshold", type=float, help="Set minimum confidence threshold for face detection", default=0.6)
@@ -29,20 +35,24 @@ parser.add_argument("--scan-every", type=int, help="Set after how many frames a
 parser.add_argument("--discard-after", type=int, help="Set the how long the tracker should keep looking for lost faces", default=10)
 parser.add_argument("--max-feature-updates", type=int, help="This is the number of seconds after which feature min/max/medium values will no longer be updated once a face has been detected.", default=900)
 parser.add_argument("--no-3d-adapt", type=int, help="When set to 1, the 3D face model will not be adapted to increase the fit", default=1)
+parser.add_argument("--try-hard", type=int, help="When set to 1, the tracker will try harder to find a face", default=0)
 parser.add_argument("--video-out", help="Set this to the filename of an AVI file to save the tracking visualization as a video", default=None)
 parser.add_argument("--video-scale", type=int, help="This is a resolution scale factor applied to the saved AVI file", default=1, choices=[1,2,3,4])
 parser.add_argument("--video-fps", type=float, help="This sets the frame rate of the output AVI file", default=24)
 parser.add_argument("--raw-rgb", type=int, help="When this is set, raw RGB frames of the size given with \"-W\" and \"-H\" are read from standard input instead of reading a video", default=0)
 parser.add_argument("--log-data", help="You can set a filename to which tracking data will be logged here", default="")
 parser.add_argument("--log-output", help="You can set a filename to console output will be logged here", default="")
-parser.add_argument("--model", type=int, help="This can be used to select the tracking model. Higher numbers are models with better tracking quality, but slower speed. Models 1 and 0 tend to be too rigid for expression and blink detection.", default=3, choices=[-1, 0, 1, 2, 3])
+parser.add_argument("--model", type=int, help="This can be used to select the tracking model. Higher numbers are models with better tracking quality, but slower speed, except for model 4, which is wink optimized. Models 1 and 0 tend to be too rigid for expression and blink detection. Model -2 is roughly equivalent to model 1, but faster. Model -3 is between models 0 and -1.", default=3, choices=[-3, -2, -1, 0, 1, 2, 3, 4])
 parser.add_argument("--model-dir", help="This can be used to specify the path to the directory containing the .onnx model files", default=None)
-parser.add_argument("--gaze-tracking", type=int, help="When set to 1, experimental blink detection and gaze tracking are enabled, which makes things slightly slower", default=1)
+parser.add_argument("--gaze-tracking", type=int, help="When set to 1, gaze tracking is enabled, which makes things slightly slower", default=1)
 parser.add_argument("--face-id-offset", type=int, help="When set, this offset is added to all face ids, which can be useful for mixing tracking data from multiple network sources", default=0)
 parser.add_argument("--repeat-video", type=int, help="When set to 1 and a video file was specified with -c, the tracker will loop the video until interrupted", default=0)
 parser.add_argument("--dump-points", type=str, help="When set to a filename, the current face 3D points are made symmetric and dumped to the given file when quitting the visualization with the \"q\" key", default="")
+parser.add_argument("--benchmark", type=int, help="When set to 1, the different tracking models are benchmarked, starting with the best and ending with the fastest and with gaze tracking disabled for models with negative IDs", default=0)
 if os.name == 'nt':
     parser.add_argument("--use-dshowcapture", type=int, help="When set to 1, libdshowcapture will be used for video input instead of OpenCV", default=1)
+    parser.add_argument("--blackmagic-options", type=str, help="When set, this additional option string is passed to the blackmagic capture library", default=None)
+    parser.add_argument("--priority", type=int, help="When set, the process priority will be changed", default=None, choices=[0, 1, 2, 3, 4, 5])
 args = parser.parse_args()
 
 os.environ["OMP_NUM_THREADS"] = str(args.max_threads)
@@ -52,12 +62,12 @@ class OutputLog(object):
         self.fh = fh
         self.output = output
     def write(self, buf):
-        if not self.fh is None:
+        if self.fh is not None:
             self.fh.write(buf)
         self.output.write(buf)
         self.flush()
     def flush(self):
-        if not self.fh is None:
+        if self.fh is not None:
             self.fh.flush()
         self.output.flush()
 output_logfile = None
@@ -66,18 +76,51 @@ if args.log_output != "":
 sys.stdout = OutputLog(output_logfile, sys.stdout)
 sys.stderr = OutputLog(output_logfile, sys.stderr)
 
-if os.name == 'nt' and args.list_cameras > 0:
+if os.name == 'nt':
     import dshowcapture
+    if args.blackmagic == 1:
+        dshowcapture.set_bm_enabled(True)
+    if args.blackmagic_options is not None:
+        dshowcapture.set_options(args.blackmagic_options)
+    if args.priority is not None:
+        import psutil
+        classes = [psutil.IDLE_PRIORITY_CLASS, psutil.BELOW_NORMAL_PRIORITY_CLASS, psutil.NORMAL_PRIORITY_CLASS, psutil.ABOVE_NORMAL_PRIORITY_CLASS, psutil.HIGH_PRIORITY_CLASS, psutil.REALTIME_PRIORITY_CLASS]
+        p = psutil.Process(os.getpid())
+        p.nice(classes[args.priority])
+
+if os.name == 'nt' and (args.list_cameras > 0 or args.list_dcaps is not None):
     cap = dshowcapture.DShowCapture()
-    camera_count = cap.get_devices()
+    info = cap.get_info()
-    if args.list_cameras == 1:
+    unit = 10000000.;
-        print("Available cameras:")
+    if args.list_dcaps is not None:
-    for i in range(camera_count):
+        formats = {0: "Any", 1: "Unknown", 100: "ARGB", 101: "XRGB", 200: "I420", 201: "NV12", 202: "YV12", 203: "Y800", 300: "YVYU", 301: "YUY2", 302: "UYVY", 303: "HDYC (Unsupported)", 400: "MJPEG", 401: "H264" }
-        camera_name = cap.get_device(i)
+        for cam in info:
+            if args.list_dcaps == -1:
+                type = ""
+                if cam['type'] == "Blackmagic":
+                    type = "Blackmagic: "
+                print(f"{cam['index']}: {type}{cam['name']}")
+            if args.list_dcaps != -1 and args.list_dcaps != cam['index']:
+                continue
+            for caps in cam['caps']:
+                format = caps['format']
+                if caps['format'] in formats:
+                    format = formats[caps['format']]
+                if caps['minCX'] == caps['maxCX'] and caps['minCY'] == caps['maxCY']:
+                    print(f"    {caps['id']}: Resolution: {caps['minCX']}x{caps['minCY']} FPS: {unit/caps['maxInterval']:.3f}-{unit/caps['minInterval']:.3f} Format: {format}")
+                else:
+                    print(f"    {caps['id']}: Resolution: {caps['minCX']}x{caps['minCY']}-{caps['maxCX']}x{caps['maxCY']} FPS: {unit/caps['maxInterval']:.3f}-{unit/caps['minInterval']:.3f} Format: {format}")
+    else:
         if args.list_cameras == 1:
-            print(f"{i}: {camera_name}")
+            print("Available cameras:")
-        else:
+        for cam in info:
-            print(camera_name)
+            type = ""
+            if cam['type'] == "Blackmagic":
+                type = "Blackmagic: "
+            if args.list_cameras == 1:
+                print(f"{cam['index']}: {type}{cam['name']}")
+            else:
+                print(f"{type}{cam['name']}")
     cap.destroy_capture()
     sys.exit(0)
 
@@ -86,8 +129,25 @@ import time
 import cv2
 import socket
 import struct
-from input_reader import InputReader, VideoReader, try_int
+import json
-from tracker import Tracker
+from input_reader import InputReader, VideoReader, DShowCaptureReader, try_int
+from tracker import Tracker, get_model_base_path
+
+if args.benchmark > 0:
+    model_base_path = get_model_base_path(args.model_dir)
+    im = cv2.imread(os.path.join(model_base_path, "benchmark.bin"), cv2.IMREAD_COLOR)
+    results = []
+    for model_type in [3, 2, 1, 0, -1, -2, -3]:
+        tracker = Tracker(224, 224, threshold=0.1, max_threads=args.max_threads, max_faces=1, discard_after=0, scan_every=0, silent=True, model_type=model_type, model_dir=args.model_dir, no_gaze=(model_type == -1), detection_threshold=0.1, use_retinaface=0, max_feature_updates=900, static_model=True if args.no_3d_adapt == 1 else False)
+        tracker.detected = 1
+        tracker.faces = [(0, 0, 224, 224)]
+        total = 0.0
+        for i in range(100):
+            start = time.perf_counter()
+            r = tracker.predict(im)
+            total += time.perf_counter() - start
+        print(1. / (total / 100.))
+    sys.exit(0)
 
 target_ip = args.ip
 target_port = args.port
@@ -95,14 +155,19 @@ target_port = args.port
 if args.faces >= 40:
     print("Transmission of tracking data over network is not supported with 40 or more faces.")
 
-fps = 0
+fps = args.fps
+dcap = None
 use_dshowcapture_flag = False
 if os.name == 'nt':
-    fps = args.fps
+    dcap = args.dcap
     use_dshowcapture_flag = True if args.use_dshowcapture == 1 else False
-    input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps, use_dshowcapture=use_dshowcapture_flag)
+    input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps, use_dshowcapture=use_dshowcapture_flag, dcap=dcap)
+    if args.dcap == -1 and type(input_reader) == DShowCaptureReader:
+        fps = min(fps, input_reader.device.get_fps())
+elif sys.platform == 'linux' and args.dformat:
+    input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps, dcap=args.dformat)
 else:
-    input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps, use_dshowcapture=use_dshowcapture_flag)
+    input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps)
 if type(input_reader.reader) == VideoReader:
     fps = 0
 
@@ -123,9 +188,9 @@ features = ["eye_l", "eye_r", "eyebrow_steepness_l", "eyebrow_updown_l", "eyebro
 if args.log_data != "":
     log = open(args.log_data, "w")
     log.write("Frame,Time,Width,Height,FPS,Face,FaceID,RightOpen,LeftOpen,AverageConfidence,Success3D,PnPError,RotationQuat.X,RotationQuat.Y,RotationQuat.Z,RotationQuat.W,Euler.X,Euler.Y,Euler.Z,RVec.X,RVec.Y,RVec.Z,TVec.X,TVec.Y,TVec.Z")
-    for i in range(66):
+    for i in range(68):
         log.write(f",Landmark[{i}].X,Landmark[{i}].Y,Landmark[{i}].Confidence")
-    for i in range(66):
+    for i in range(70):
         log.write(f",Point3D[{i}].X,Point3D[{i}].Y,Point3D[{i}].Z")
     for feature in features:
         log.write(f",{feature}")
@@ -146,7 +211,7 @@ try:
     source_name = input_reader.name
     while repeat or input_reader.is_open():
         if not input_reader.is_open() or need_reinit == 1:
-            input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps, use_dshowcapture=use_dshowcapture_flag)
+            input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps, use_dshowcapture=use_dshowcapture_flag, dcap=dcap)
             if input_reader.name != source_name:
                 print(f"Failed to reinitialize camera and got {input_reader.name} instead of {source_name}.")
                 sys.exit(1)
@@ -158,6 +223,8 @@ try:
             continue
 
         ret, frame = input_reader.read()
+        if ret and args.mirror_input:
+            frame = cv2.flip(frame, 1)
         if not ret:
             if repeat:
                 if need_reinit == 0:
@@ -184,8 +251,8 @@ try:
             first = False
             height, width, channels = frame.shape
             sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-            tracker = Tracker(width, height, threshold=args.threshold, max_threads=args.max_threads, max_faces=args.faces, discard_after=args.discard_after, scan_every=args.scan_every, silent=False if args.silent == 0 else True, model_type=args.model, model_dir=args.model_dir, no_gaze=False if args.gaze_tracking != 0 else True, detection_threshold=args.detection_threshold, use_retinaface=args.scan_retinaface, max_feature_updates=args.max_feature_updates, static_model=True if args.no_3d_adapt == 1 else False)
+            tracker = Tracker(width, height, threshold=args.threshold, max_threads=args.max_threads, max_faces=args.faces, discard_after=args.discard_after, scan_every=args.scan_every, silent=False if args.silent == 0 else True, model_type=args.model, model_dir=args.model_dir, no_gaze=False if args.gaze_tracking != 0 and args.model != -1 else True, detection_threshold=args.detection_threshold, use_retinaface=args.scan_retinaface, max_feature_updates=args.max_feature_updates, static_model=True if args.no_3d_adapt == 1 else False, try_hard=args.try_hard == 1)
-            if not args.video_out is None:
+            if args.video_out is not None:
                 out = cv2.VideoWriter(args.video_out, cv2.VideoWriter_fourcc('F','F','V','1'), args.video_fps, (width * args.video_scale, height * args.video_scale))
 
         try:
@@ -228,8 +295,8 @@ try:
                 packet.extend(bytearray(struct.pack("f", f.translation[0])))
                 packet.extend(bytearray(struct.pack("f", f.translation[1])))
                 packet.extend(bytearray(struct.pack("f", f.translation[2])))
-                if not log is None:
+                if log is not None:
-                    log.write(f"{frame_count},{now},{width},{height},{args.fps},{face_num},{f.id},{f.eye_blink[0]},{f.eye_blink[1]},{f.conf},{f.success},{f.pnp_error},{f.quaternion[0]},{f.quaternion[1]},{f.quaternion[2]},{f.quaternion[3]},{f.euler[0]},{f.euler[1]},{f.euler[2]},{f.rotation[0]},{f.rotation[1]},{f.rotation[2]},{f.translation[0]},{f.translation[1]},{f.translation[2]}")
+                    log.write(f"{frame_count},{now},{width},{height},{fps},{face_num},{f.id},{f.eye_blink[0]},{f.eye_blink[1]},{f.conf},{f.success},{f.pnp_error},{f.quaternion[0]},{f.quaternion[1]},{f.quaternion[2]},{f.quaternion[3]},{f.euler[0]},{f.euler[1]},{f.euler[2]},{f.rotation[0]},{f.rotation[1]},{f.rotation[2]},{f.translation[0]},{f.translation[1]},{f.translation[2]}")
                 for (x,y,c) in f.lms:
                     packet.extend(bytearray(struct.pack("f", c)))
                 if args.visualize > 1:
@@ -239,32 +306,23 @@ try:
                 for pt_num, (x,y,c) in enumerate(f.lms):
                     packet.extend(bytearray(struct.pack("f", y)))
                     packet.extend(bytearray(struct.pack("f", x)))
-                    if not log is None:
+                    if log is not None:
                         log.write(f",{y},{x},{c}")
-                    if pt_num == 66 and (f.eye_blink[0] < 0.30 or c < 0.30):
+                    if pt_num == 66 and (f.eye_blink[0] < 0.30 or c < 0.20):
                         continue
-                    if pt_num == 67 and (f.eye_blink[1] < 0.30 or c < 0.30):
+                    if pt_num == 67 and (f.eye_blink[1] < 0.30 or c < 0.20):
                         continue
                     x = int(x + 0.5)
                     y = int(y + 0.5)
-                    if args.visualize != 0 or not out is None:
+                    if args.visualize != 0 or out is not None:
                         if args.visualize > 3:
                             frame = cv2.putText(frame, str(pt_num), (int(y), int(x)), cv2.FONT_HERSHEY_SIMPLEX, 0.25, (255,255,0))
                         color = (0, 255, 0)
                         if pt_num >= 66:
                             color = (255, 255, 0)
                         if not (x < 0 or y < 0 or x >= height or y >= width):
-                            frame[int(x), int(y)] = color
+                            cv2.circle(frame, (y, x), 1, color, -1)
-                        x += 1
+                if args.pnp_points != 0 and (args.visualize != 0 or out is not None) and f.rotation is not None:
-                        if not (x < 0 or y < 0 or x >= height or y >= width):
-                            frame[int(x), int(y)] = color
-                        y += 1
-                        if not (x < 0 or y < 0 or x >= height or y >= width):
-                            frame[int(x), int(y)] = color
-                        x -= 1
-                        if not (x < 0 or y < 0 or x >= height or y >= width):
-                            frame[int(x), int(y)] = color
-                if args.pnp_points != 0 and (args.visualize != 0 or not out is None) and f.rotation is not None:
                     if args.pnp_points > 1:
                         projected = cv2.projectPoints(f.face_3d[0:66], f.rotation, f.translation, tracker.camera, tracker.dist_coeffs)
                     else:
@@ -287,7 +345,7 @@ try:
                     packet.extend(bytearray(struct.pack("f", x)))
                     packet.extend(bytearray(struct.pack("f", -y)))
                     packet.extend(bytearray(struct.pack("f", -z)))
-                    if not log is None:
+                    if log is not None:
                         log.write(f",{x},{-y},{-z}")
                 if f.current_features is None:
                     f.current_features = {}
@@ -295,16 +353,16 @@ try:
                     if not feature in f.current_features:
                         f.current_features[feature] = 0
                     packet.extend(bytearray(struct.pack("f", f.current_features[feature])))
-                    if not log is None:
+                    if log is not None:
                         log.write(f",{f.current_features[feature]}")
-                if not log is None:
+                if log is not None:
                     log.write("\r\n")
                     log.flush()
 
             if detected and len(faces) < 40:
                 sock.sendto(packet, (target_ip, target_port))
 
-            if not out is None:
+            if out is not None:
                 video_frame = frame
                 if args.video_scale != 1:
                     video_frame = cv2.resize(frame, (width * args.video_scale, height * args.video_scale), interpolation=cv2.INTER_NEAREST)
@@ -315,7 +373,7 @@ try:
             if args.visualize != 0:
                 cv2.imshow('OpenSeeFace Visualization', frame)
                 if cv2.waitKey(1) & 0xFF == ord('q'):
-                    if args.dump_points != "" and not faces is None and len(faces) > 0:
+                    if args.dump_points != "" and faces is not None and len(faces) > 0:
                         np.set_printoptions(threshold=sys.maxsize, precision=15)
                         pairs = [
                             (0, 16),
@@ -391,9 +449,10 @@ except KeyboardInterrupt:
         print("Quitting")
 
 input_reader.close()
-if not out is None:
+if out is not None:
     out.release()
-cv2.destroyAllWindows()
+if args.visualize != 0:
+    cv2.destroyAllWindows()
 
 if args.silent == 0 and tracking_frames > 0:
     average_tracking_time = 1000 * tracking_time / tracking_frames

facetracker.spec

@@ -5,7 +5,7 @@ block_cipher = None
 
 a = Analysis(['facetracker.py'],
              pathex=['C:\\OpenSeeFaceBuild'],
-             binaries=[('dshowcapture/dshowcapture_x86.dll', '.'), ('dshowcapture/dshowcapture_x64.dll', '.'), ('escapi/escapi_x86.dll', '.'), ('escapi/escapi_x64.dll', '.'), ('run.bat', '.'), ('c:/windows/system32/msvcp140.dll', '.'), ('c:/windows/system32/vcomp140.dll', '.'), ('c:/windows/system32/concrt140.dll', '.'), ('c:/windows/system32/vccorlib140.dll', '.')],
+             binaries=[('dshowcapture/dshowcapture_x86.dll', '.'), ('dshowcapture/dshowcapture_x64.dll', '.'), ('dshowcapture/libminibmcapture32.dll', '.'), ('dshowcapture/libminibmcapture64.dll', '.'), ('escapi/escapi_x86.dll', '.'), ('escapi/escapi_x64.dll', '.'), ('run.bat', '.'), ('msvcp140.dll', '.'), ('vcomp140.dll', '.'), ('concrt140.dll', '.'), ('vccorlib140.dll', '.')],
              datas=[],
              hiddenimports=[],
              hookspath=[],

input_reader.py

@@ -36,7 +36,7 @@ class EscapiReader(VideoReader):
         self.fps = fps
         self.device = capture
         escapi.count_capture_devices()
-        self.name = str(escapi.device_name(self.device).decode('utf8'))
+        self.name = str(escapi.device_name(self.device).decode('utf8', 'surrogateescape'))
         self.buffer = escapi.init_camera(self.device, self.width, self.height, self.fps)
         escapi.do_capture(self.device)
     def is_open(self):
@@ -54,16 +54,34 @@ class EscapiReader(VideoReader):
         escapi.deinit_camera(self.device)
 
 class DShowCaptureReader(VideoReader):
-    def __init__(self, capture, width, height, fps, use_dshowcapture=True):
+    def __init__(self, capture, width, height, fps, use_dshowcapture=True, dcap=None):
         self.device = None
         self.width = width
         self.height = height
         self.fps = fps
+        self.dcap = dcap;
         self.device = dshowcapture.DShowCapture()
         self.device.get_devices()
-        self.name = self.device.get_device(capture)
+        info = self.device.get_info()
-        self.device.capture_device(capture, self.width, self.height, self.fps)
+        self.name = info[capture]['name']
-        print(f"Camera: \"{self.name}\" Resolution: {self.device.get_width()}x{self.device.get_height()} Frame rate: {self.device.get_fps()} Colorspace: {self.device.get_colorspace()} Flipped: {self.device.get_flipped()}")
+        if info[capture]['type'] == "Blackmagic":
+            self.name = "Blackmagic: " + self.name
+            if dcap is None or dcap < 0:
+                dcap = 0
+        ret = False
+        if dcap is None:
+            ret = self.device.capture_device(capture, self.width, self.height, self.fps)
+        else:
+            if dcap < 0:
+                ret = self.device.capture_device_default(capture)
+            else:
+                ret = self.device.capture_device_by_dcap(capture, dcap, self.width, self.height, self.fps)
+        if not ret:
+            raise Exception("Failed to start capture.")
+        self.width = self.device.width
+        self.height = self.device.height
+        self.fps = self.device.fps
+        print(f"Camera: \"{self.name}\" Capability ID: {dcap} Resolution: {self.device.width}x{self.device.height} Frame rate: {self.device.fps} Colorspace: {self.device.colorspace} Internal: {self.device.colorspace_internal} Flipped: {self.device.flipped}")
         self.timeout = 1000
     def is_open(self):
         return self.device.capturing()
@@ -93,6 +111,7 @@ class OpenCVReader(VideoReader):
         super(OpenCVReader, self).__init__(capture, camera=True)
         self.cap.set(3, width)
         self.cap.set(4, height)
+        self.cap.set(38, 1)
     def is_open(self):
         return super(OpenCVReader, self).is_open()
     def is_ready(self):
@@ -102,6 +121,30 @@ class OpenCVReader(VideoReader):
     def close(self):
         super(OpenCVReader, self).close()
 
+class V4L2Reader(OpenCVReader):
+    def __init__(self, capture, width, height, fps, dformat):
+        self.cap = cv2.VideoCapture(capture, cv2.CAP_V4L2)
+        if dformat:
+            fourcc = cv2.VideoWriter.fourcc(*dformat)
+            self.cap.set(6, fourcc)
+        self.device = None
+        self.width = width
+        self.height = height
+        self.fps = fps
+        self.name = str(capture)
+        self.cap.set(3, width)
+        self.cap.set(4, height)
+        self.cap.set(38, 1)
+        self.cap.set(5, fps)
+    def is_open(self):
+        return super(V4L2Reader, self).is_open()
+    def is_ready(self):
+        return super(V4L2Reader, self).is_ready()
+    def read(self):
+        return super(V4L2Reader, self).read()
+    def close(self):
+        super(V4L2Reader, self).close()
+
 class RawReader:
     def __init__(self, width, height):
         self.width = int(width)
@@ -159,7 +202,7 @@ def test_reader(reader):
         return False
 
 class InputReader():
-    def __init__(self, capture, raw_rgb, width, height, fps, use_dshowcapture=False):
+    def __init__(self, capture, raw_rgb, width, height, fps, use_dshowcapture=False, dcap=None):
         self.reader = None
         self.name = str(capture)
         try:
@@ -174,7 +217,7 @@ class InputReader():
                     name = ""
                     try:
                         if use_dshowcapture:
-                            self.reader = DShowCaptureReader(int(capture), width, height, fps)
+                            self.reader = DShowCaptureReader(int(capture), width, height, fps, dcap=dcap)
                             name = self.reader.name
                             good = test_reader(self.reader)
                             self.name = name
@@ -194,7 +237,7 @@ class InputReader():
                         devices = escapi.count_capture_devices()
                         found = None
                         for i in range(devices):
-                            escapi_name = str(escapi.device_name(i).decode('utf8'))
+                            escapi_name = str(escapi.device_name(i).decode('utf8', 'surrogateescape'))
                             if name == escapi_name:
                                 found = i
                         if found is None:
@@ -213,6 +256,14 @@ class InputReader():
                     print(f"Escapi failed. Falling back to OpenCV. If this fails, please change your camera settings.", file=sys.stderr)
                     self.reader = OpenCVReader(int(capture), width, height, fps)
                     self.name = self.reader.name
+                elif sys.platform == 'linux':
+                    try:
+                        self.reader = V4L2Reader(int(capture), width, height, fps, dcap)
+                    except:
+                        print("V4L2 exception: ")
+                        traceback.print_exc()
+                        print(f"V4L2 failed. Falling back to OpenCV. If this fails, please change your camera settings.", file=sys.stderr)
+                        self.reader = OpenCVReader(int(capture), width, height, fps)
                 else:
                     self.reader = OpenCVReader(int(capture), width, height, fps)
         except Exception as e:

make_exe.bat

@@ -1,25 +1,31 @@
-REM pyinstaller facetracker.spec --onefile ^
+echo "Started"
-REM     --add-binary dshowcapture/dshowcapture_x86.dll;. ^
-REM     --add-binary dshowcapture/dshowcapture_x64.dll;. ^
-REM     --add-binary escapi/escapi_x86.dll;. ^
-REM     --add-binary escapi/escapi_x64.dll;. ^
-REM     --add-binary msvcp140.dll;. ^
-REM     --add-binary vcomp140.dll;. ^
-REM     --add-binary concrt140.dll;. ^
-REM     --add-binary vccorlib140.dll;. ^
-REM     --add-binary run.bat;.
 
-pyinstaller facetracker.spec --onedir ^
+:: A venv is used so it's easy to add the onnxruntime dll to the binary
-    --add-binary dshowcapture/dshowcapture_x86.dll;. ^
+
-    --add-binary dshowcapture/dshowcapture_x64.dll;. ^
+echo "Creating venv"
-    --add-binary escapi/escapi_x86.dll;. ^
+python -m venv venv
-    --add-binary escapi/escapi_x64.dll;. ^
+
+echo "Activating venv"
+call venv\Scripts\activate.bat
+
+echo "Installing dependencies"
+pip install wheel
+
+pip install onnxruntime opencv-python==4.5.4.60 pillow numpy==1.23.0 pyinstaller
+
+echo "Running pyinstaller"
+pyinstaller facetracker.py --clean ^
+    --onedir ^
+    --add-binary dshowcapture/*.dll;. ^
+    --add-binary escapi/*.dll;. ^
+    --add-binary venv/lib/site-packages/onnxruntime/capi/*.dll;onnxruntime\capi ^
     --add-binary msvcp140.dll;. ^
     --add-binary vcomp140.dll;. ^
     --add-binary concrt140.dll;. ^
     --add-binary vccorlib140.dll;. ^
     --add-binary run.bat;.
 
-del dist\facetracker\cv2\opencv_videoio_ffmpeg420_64.dll
+echo "Deleting opencv dll"
+del dist\facetracker\cv2\opencv_videoio_*
 
-pause
+echo "Finished"

make_exe.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+
+PYTHON_BINARY=$(which python3)
+
+if [[ -z "$PYTHON_BINARY" ]]; then
+    echo "python3 not found. Searching for python instead."
+    PYTHON_BINARY=$(which python)
+    if [[ -z "$PYTHON_BINARY" ]]; then
+        echo "Python 3 must be available on your path. Exiting."
+        exit 1
+    fi
+fi
+
+echo "Creating venv"
+"$PYTHON_BINARY" -m venv venv
+
+source venv/bin/activate
+
+echo "Installing packages"
+pip install wheel # Make sure this is installed beforehand
+
+pip install onnxruntime opencv-python==4.5.4.60 pillow numpy==1.23.0 pyinstaller
+
+echo "Creating binary"
+pyinstaller --onedir --clean facetracker.py \
+    --add-binary venv/lib/python3.*/site-packages/onnxruntime/capi/*.so:onnxruntime/capi
+
+echo "Files should be available in the dist/ folder"
+echo "Done!"
+

model.py

@@ -1,7 +1,7 @@
 # This file is not used by the tracking application and currently outdated
 import torch
 import torch.nn as nn
-import geffnet.mobilenetv3 # geffnet.mobilenetv3._gen_mobilenet_v3 needs to be patched to return the parameters instead of instantiating the network
+import geffnet.mobilenetv3 # geffnet.mobilenetv3._gen_mobilenet_v3 needs to be patched to return the parameters instead of instantiating the network, based on commit c450c12ae6ffb1757f62dde3c2765da3c10f6def
 from geffnet.efficientnet_builder import round_channels
 
 class DSConv2d(nn.Module):
@@ -121,6 +121,10 @@ class OpenSeeFaceDetect(geffnet.mobilenetv3.MobileNetV3):
     def forward(self, x):
         return self._forward_impl(x)
 
+def logit_arr(p, factor=16.0):
+    p = p.clamp(0.0000001, 0.9999999)
+    return torch.log(p / (1 - p)) / factor
+
 # Landmark detection model
 # Models:
 # 0: "small", 0.5
@@ -161,17 +165,17 @@ class OpenSeeFaceLandmarks(geffnet.mobilenetv3.MobileNetV3):
         x = self.group(x)
 
         if self.inference:
-            t_main = x[0:66].reshape((66, 28*28))
+            t_main = x[:, 0:66].reshape((-1, 66, 28*28))
-            t_m = t_main.argmax(dim=1)
+            t_m = t_main.argmax(dim=2)
-            indices = t_m.unsqueeze(1)
+            indices = t_m.unsqueeze(2)
-            t_conf = t_main.gather(1, indices).squeeze(1)
+            t_conf = t_main.gather(2, indices).squeeze(2)
-            t_off_x = x[66:132].reshape((66, 28*28)).gather(1, indices).squeeze(1)
+            t_off_x = x[:, 66:132].reshape((-1, 66, 28*28)).gather(2, indices).squeeze(2)
-            t_off_y = x[132:198].reshape((66, 28*28)).gather(1, indices).squeeze(1)
+            t_off_y = x[:, 132:198].reshape((-1, 66, 28*28)).gather(2, indices).squeeze(2)
             t_off_x = (223. * logit_arr(t_off_x) + 0.5).floor()
             t_off_y = (223. * logit_arr(t_off_y) + 0.5).floor()
             t_x = 223. * (t_m / 28.).floor() / 27. + t_off_x
             t_y = 223. * t_m.remainder(28.).float() / 27. + t_off_y
-            x = (t_conf.mean(), torch.stack([t_x, t_y, t_conf], 1))
+            x = (t_conf.mean(1), torch.stack([t_x, t_y, t_conf], 2))
 
         return x
     def forward(self, x):
@@ -203,17 +207,17 @@ class OpenSeeFaceLandmarks30Pt(geffnet.mobilenetv3.MobileNetV3):
         x = self.group(x)
 
         if self.inference:
-            t_main = x[0:30].reshape((30, 7*7))
+            t_main = x[:, 0:30].reshape((-1, 30, 7*7))
-            t_m = t_main.argmax(dim=1)
+            t_m = t_main.argmax(dim=2)
-            indices = t_m.unsqueeze(1)
+            indices = t_m.unsqueeze(2)
-            t_conf = t_main.gather(1, indices).squeeze(1)
+            t_conf = t_main.gather(2, indices).squeeze(2)
-            t_off_x = x[30:60].reshape((60, 7*7)).gather(1, indices).squeeze(1)
+            t_off_x = x[:, 30:60].reshape((-1, 30, 7*7)).gather(2, indices).squeeze(2)
-            t_off_y = x[60:90].reshape((60, 7*7)).gather(1, indices).squeeze(1)
+            t_off_y = x[:, 60:90].reshape((-1, 30, 7*7)).gather(2, indices).squeeze(2)
-            t_off_x = 55. * logit_arr(t_off_x)
+            t_off_x = 55. * logit_arr(t_off_x, factor=8.0)
-            t_off_y = 55. * logit_arr(t_off_y)
+            t_off_y = 55. * logit_arr(t_off_y, factor=8.0)
             t_x = 55. * (t_m / 7.).floor() / 6. + t_off_x
             t_y = 55. * t_m.remainder(7.).float() / 6. + t_off_y
-            x = (t_conf.mean(), torch.stack([t_x, t_y, t_conf], 1))
+            x = (t_conf.mean(1), torch.stack([t_x, t_y, t_conf], 2))
 
         return x
     def forward(self, x):
@@ -247,7 +251,7 @@ def AdapWingLoss(pre_hm, gt_hm):
         img_dilate = cv2.morphologyEx(img_merge, cv2.MORPH_DILATE, kernel)
         img_dilate[img_dilate < 51] = 1  # 0*W+1
         img_dilate[img_dilate >= 51] = 11  # 1*W+1
-        img_dilate = np.array(img_dilate, dtype=np.int)
+        img_dilate = np.array(img_dilate, dtype=int)
         img_dilate = img_dilate.transpose(2, 0, 1)
         dilated = torch.from_numpy(img_dilate).float()
         if first:

poetry.lock

@@ -0,0 +1,224 @@
+[[package]]
+name = "flatbuffers"
+version = "2.0"
+description = "The FlatBuffers serialization format for Python"
+category = "main"
+optional = false
+python-versions = "*"
+
+[[package]]
+name = "numpy"
+version = "1.22"
+description = "NumPy is the fundamental package for array computing with Python."
+category = "main"
+optional = false
+python-versions = ">=3.7,<3.11"
+
+[[package]]
+name = "onnxruntime"
+version = "1.9.0"
+description = "ONNX Runtime is a runtime accelerator for Machine Learning models"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+flatbuffers = "*"
+numpy = ">=1.16.6"
+protobuf = "*"
+
+[[package]]
+name = "opencv-python"
+version = "4.5.4.58"
+description = "Wrapper package for OpenCV python bindings."
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+numpy = ">=1.21.2"
+
+[[package]]
+name = "pillow"
+version = "10.2.0"
+description = "Python Imaging Library (Fork)"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[[package]]
+name = "protobuf"
+version = "4.25.8"
+description = "Protocol Buffers"
+category = "main"
+optional = false
+python-versions = ">=3.5"
+
+[metadata]
+lock-version = "1.1"
+python-versions = ">=3.7,<3.11"
+content-hash = "1511681290c580145b1117beffe728052d856eee3ad8419930626c6fc6c9a09d"
+
+[metadata.files]
+flatbuffers = [
+    {file = "flatbuffers-2.0-py2.py3-none-any.whl", hash = "sha256:3751954f0604580d3219ae49a85fafec9d85eec599c0b96226e1bc0b48e57474"},
+    {file = "flatbuffers-2.0.tar.gz", hash = "sha256:12158ab0272375eab8db2d663ae97370c33f152b27801fa6024e1d6105fd4dd2"},
+]
+numpy = [
+    {file = "numpy-1.21.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:508b0b513fa1266875524ba8a9ecc27b02ad771fe1704a16314dc1a816a68737"},
+    {file = "numpy-1.21.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5dfe9d6a4c39b8b6edd7990091fea4f852888e41919d0e6722fe78dd421db0eb"},
+    {file = "numpy-1.21.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a10968963640e75cc0193e1847616ab4c718e83b6938ae74dea44953950f6b7"},
+    {file = "numpy-1.21.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49c6249260890e05b8111ebfc391ed58b3cb4b33e63197b2ec7f776e45330721"},
+    {file = "numpy-1.21.3-cp310-cp310-win_amd64.whl", hash = "sha256:f8f4625536926a155b80ad2bbff44f8cc59e9f2ad14cdda7acf4c135b4dc8ff2"},
+    {file = "numpy-1.21.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e54af82d68ef8255535a6cdb353f55d6b8cf418a83e2be3569243787a4f4866f"},
+    {file = "numpy-1.21.3-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f41b018f126aac18583956c54544db437f25c7ee4794bcb23eb38bef8e5e192a"},
+    {file = "numpy-1.21.3-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:50cd26b0cf6664cb3b3dd161ba0a09c9c1343db064e7c69f9f8b551f5104d654"},
+    {file = "numpy-1.21.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cc9b512e9fb590797474f58b7f6d1f1b654b3a94f4fa8558b48ca8b3cfc97cf"},
+    {file = "numpy-1.21.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:88a5d6b268e9ad18f3533e184744acdaa2e913b13148160b1152300c949bbb5f"},
+    {file = "numpy-1.21.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:3c09418a14471c7ae69ba682e2428cae5b4420a766659605566c0fa6987f6b7e"},
+    {file = "numpy-1.21.3-cp37-cp37m-win32.whl", hash = "sha256:90bec6a86b348b4559b6482e2b684db4a9a7eed1fa054b86115a48d58fbbf62a"},
+    {file = "numpy-1.21.3-cp37-cp37m-win_amd64.whl", hash = "sha256:043e83bfc274649c82a6f09836943e4a4aebe5e33656271c7dbf9621dd58b8ec"},
+    {file = "numpy-1.21.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:75621882d2230ab77fb6a03d4cbccd2038511491076e7964ef87306623aa5272"},
+    {file = "numpy-1.21.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:188031f833bbb623637e66006cf75e933e00e7231f67e2b45cf8189612bb5dc3"},
+    {file = "numpy-1.21.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:160ccc1bed3a8371bf0d760971f09bfe80a3e18646620e9ded0ad159d9749baa"},
+    {file = "numpy-1.21.3-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:29fb3dcd0468b7715f8ce2c0c2d9bbbaf5ae686334951343a41bd8d155c6ea27"},
+    {file = "numpy-1.21.3-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:32437f0b275c1d09d9c3add782516413e98cd7c09e6baf4715cbce781fc29912"},
+    {file = "numpy-1.21.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e606e6316911471c8d9b4618e082635cfe98876007556e89ce03d52ff5e8fcf0"},
+    {file = "numpy-1.21.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a99a6b067e5190ac6d12005a4d85aa6227c5606fa93211f86b1dafb16233e57d"},
+    {file = "numpy-1.21.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:dde972a1e11bb7b702ed0e447953e7617723760f420decb97305e66fb4afc54f"},
+    {file = "numpy-1.21.3-cp38-cp38-win32.whl", hash = "sha256:fe52dbe47d9deb69b05084abd4b0df7abb39a3c51957c09f635520abd49b29dd"},
+    {file = "numpy-1.21.3-cp38-cp38-win_amd64.whl", hash = "sha256:75eb7cadc8da49302f5b659d40ba4f6d94d5045fbd9569c9d058e77b0514c9e4"},
+    {file = "numpy-1.21.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2a6ee9620061b2a722749b391c0d80a0e2ae97290f1b32e28d5a362e21941ee4"},
+    {file = "numpy-1.21.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5c4193f70f8069550a1788bd0cd3268ab7d3a2b70583dfe3b2e7f421e9aace06"},
+    {file = "numpy-1.21.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:28f15209fb535dd4c504a7762d3bc440779b0e37d50ed810ced209e5cea60d96"},
+    {file = "numpy-1.21.3-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c6c2d535a7beb1f8790aaa98fd089ceab2e3dd7ca48aca0af7dc60e6ef93ffe1"},
+    {file = "numpy-1.21.3-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:bffa2eee3b87376cc6b31eee36d05349571c236d1de1175b804b348dc0941e3f"},
+    {file = "numpy-1.21.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc14e7519fab2a4ed87d31f99c31a3796e4e1fe63a86ebdd1c5a1ea78ebd5896"},
+    {file = "numpy-1.21.3-cp39-cp39-win32.whl", hash = "sha256:dd0482f3fc547f1b1b5d6a8b8e08f63fdc250c58ce688dedd8851e6e26cff0f3"},
+    {file = "numpy-1.21.3-cp39-cp39-win_amd64.whl", hash = "sha256:300321e3985c968e3ae7fbda187237b225f3ffe6528395a5b7a5407f73cf093e"},
+    {file = "numpy-1.21.3-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98339aa9911853f131de11010f6dd94c8cec254d3d1f7261528c3b3e3219f139"},
+    {file = "numpy-1.21.3.zip", hash = "sha256:63571bb7897a584ca3249c86dd01c10bcb5fe4296e3568b2e9c1a55356b6410e"},
+]
+onnxruntime = [
+    {file = "onnxruntime-1.9.0-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:48f0fcf3c9aa6836584e64abe63fa7395c02066d3259bbdeb489b4d172e0127a"},
+    {file = "onnxruntime-1.9.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f9d772a6330cb85e7723f84e357320a1603e3824a92aab4ef36fc3a41e64f16"},
+    {file = "onnxruntime-1.9.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3bdb861822a63404cca7b46dce86d48bbc21c906a4b4ed13969bc89763ac7f96"},
+    {file = "onnxruntime-1.9.0-cp36-cp36m-win32.whl", hash = "sha256:e3f8f7d5d4d66e3a4a2b731a000d3142a53a5403e8814e68bbd659514e815899"},
+    {file = "onnxruntime-1.9.0-cp36-cp36m-win_amd64.whl", hash = "sha256:cf3edbc54bfe99a119d73cd65398a2ec68ae3af2557ab7e645976314a8d11aa1"},
+    {file = "onnxruntime-1.9.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:c8ff9c914b2b1c3b022dedc199e3f971e340d8923a1ef42d66530508fa367bf6"},
+    {file = "onnxruntime-1.9.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d03ff4a2717c4149acc7c649fd66a67e81ec44c9e6e2a00df1d6e9ca843f1b7"},
+    {file = "onnxruntime-1.9.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8a2315e2244ac371742f6e30da5367c680c3e84c31e291a35f8ddfab09c3c82"},
+    {file = "onnxruntime-1.9.0-cp37-cp37m-win32.whl", hash = "sha256:fa927b1825f2851c0c8f3948515a56d76cb0686da9acd1d6f8fafe552c8d8fec"},
+    {file = "onnxruntime-1.9.0-cp37-cp37m-win_amd64.whl", hash = "sha256:9ccaf6a0365f2b86efe21681416b8cfe97f084a7d53bd1cf2bf889a0aef2b0d3"},
+    {file = "onnxruntime-1.9.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:d20ce3448babe89a77cc9d357730767deb3617e36439bddcd006f28abc72b416"},
+    {file = "onnxruntime-1.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3fd1d6647245aa38e1099cfd355d84e807de5350d5216e84ceefd91c64ce243d"},
+    {file = "onnxruntime-1.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f00620fc0f51bc4d90ae6d96ceb4b6538e3bd1e328178104118ac672f37c40d"},
+    {file = "onnxruntime-1.9.0-cp38-cp38-win32.whl", hash = "sha256:e1c1fe3f7d960eeffc02a5f196d85529254eefd59cbeecd8abee0a9467b5c2d8"},
+    {file = "onnxruntime-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:9002214af1b2317ab3a63a2f045f7d1363c207e661d475a877aa6499ca09d606"},
+    {file = "onnxruntime-1.9.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:7339cef9b918b88f1fec8109cfa0a8416f119c5968d00300a9186847d86e35de"},
+    {file = "onnxruntime-1.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdec8538eb59e63a376d0677f7ec043ceb597d52ee88f1f7e250928893a0de7f"},
+    {file = "onnxruntime-1.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab62b29429e0e62c11478b2a8a3af2646531fba7800736e8b201d8baa50b43a8"},
+    {file = "onnxruntime-1.9.0-cp39-cp39-win32.whl", hash = "sha256:4aee9a893f93637341fd0e6b56fa3ab1c430d718d08d79a358603297a1575ad9"},
+    {file = "onnxruntime-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:bdf1327932227383b04093a51266474b2703b3fcf9c0f6f11c652d9652b76a5c"},
+]
+opencv-python = [
+    {file = "opencv-python-4.5.4.58.tar.gz", hash = "sha256:48288428f407bacba5f73d460feb4a1ecafe87db3d7cfc0730a49fb32f589bbf"},
+    {file = "opencv_python-4.5.4.58-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0eba0bfe62c48a02a5af3a0944e872c99f57f98653bed14d51c6991a58f9e1d1"},
+    {file = "opencv_python-4.5.4.58-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:9bcca50c5444b5cfb01624666b69f91ba8f2d2bf4ef37b111697aafdeb81c99f"},
+    {file = "opencv_python-4.5.4.58-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:8f7886acabaebf0361bd3dbccaa0d08e3f65ab13b7c739eb11e028f01ad13582"},
+    {file = "opencv_python-4.5.4.58-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:d4b1d0b98ee72ba5dd720166790fc93ce459281e138ee79b0d41420b3da52b2e"},
+    {file = "opencv_python-4.5.4.58-cp310-cp310-win32.whl", hash = "sha256:69a78e40a374ac14e4bf15a13dbb6c30fd2fbd5fcd3674d020a31b88861d5aaf"},
+    {file = "opencv_python-4.5.4.58-cp310-cp310-win_amd64.whl", hash = "sha256:315c357522b6310ef7a0718d9f0c5d3110e59c19140705499a3c29bdd8c0124f"},
+    {file = "opencv_python-4.5.4.58-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:887a61097092dc0bf23fa24646dbc8cfeeb753649cb28a3782a93a6879e3b7d2"},
+    {file = "opencv_python-4.5.4.58-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:22bcc3153a7d4f95aff79457eef81ef5e40ab1851b189e014412b5e9fbee2573"},
+    {file = "opencv_python-4.5.4.58-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:92e9b2261ec764229c948d77fe0d922ee033348ca6519939b87861016c1614b3"},
+    {file = "opencv_python-4.5.4.58-cp36-cp36m-win32.whl", hash = "sha256:0d6249a49122a78afc6685ddb1377a87e46414ae61c84535c4c6024397f1f3e8"},
+    {file = "opencv_python-4.5.4.58-cp36-cp36m-win_amd64.whl", hash = "sha256:eaa144013b597e4dcabc8d8230edfe810319de01b5609556d415a20e2b707547"},
+    {file = "opencv_python-4.5.4.58-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:26feeeb280de179f5dbb8976ebf7ceb836bd263973cb5daec8ca36e8ef7b5773"},
+    {file = "opencv_python-4.5.4.58-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:4a13381bdfc0fb4b080efcc27c46561d0bd752f126226e9f19aa9cbcf6677f40"},
+    {file = "opencv_python-4.5.4.58-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:ac852fcaac93439f2f7116ddffdc23fd366c872200ade2272446f9898180cecb"},
+    {file = "opencv_python-4.5.4.58-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:02872e0a9358526646d691f390143e9c21109c210095314abaa0641211cda077"},
+    {file = "opencv_python-4.5.4.58-cp37-cp37m-win32.whl", hash = "sha256:6b87bab220d17e03eeedbcc6652d9d7e7bb09886dbd0f810310697a948b4c6fd"},
+    {file = "opencv_python-4.5.4.58-cp37-cp37m-win_amd64.whl", hash = "sha256:a2a7f09b8843b85f3e1b02c5ea3ddc0cb9f5ad9698380109b37069ee8db7746d"},
+    {file = "opencv_python-4.5.4.58-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:c44f5c51e92322ed832607204249c190764dec6cf29e8ba6d679b10326be1c1b"},
+    {file = "opencv_python-4.5.4.58-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9b2c198af083a693d42a82bddc4d1f7e6bb02c64192ff7fac1fd1d43a8cf1be6"},
+    {file = "opencv_python-4.5.4.58-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:637f4d3ad81bd27f273ede4c5fa6c26afb85c097c9715baf107cc270e37f5fea"},
+    {file = "opencv_python-4.5.4.58-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:2fff48a641a74d1def31c1e88f9e5ce50ba4d0f87d085dfbf8bc844e12f6cd54"},
+    {file = "opencv_python-4.5.4.58-cp38-cp38-win32.whl", hash = "sha256:8ddf4dcd8199209e33f21deb0c6d8ab62b21802816bba895fefc346b6d2e522d"},
+    {file = "opencv_python-4.5.4.58-cp38-cp38-win_amd64.whl", hash = "sha256:085c5fcf5a6479c34aca3fd0f59055e704083d6a44009d6583c675ff1a5a0625"},
+    {file = "opencv_python-4.5.4.58-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:4abe9c4fb6fe16daa9fcdd68b5357d3530431341aa655203f8e84f394e1fe6d4"},
+    {file = "opencv_python-4.5.4.58-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4b614fbd81aeda53ce28e645aaee18fda7c7f2a48eb7f1a70a7c6c3427946342"},
+    {file = "opencv_python-4.5.4.58-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:215bdf069847d4e3b0447a34e9eb4046dd4ca523d41fe4381c1c55f6704fd0dc"},
+    {file = "opencv_python-4.5.4.58-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc34cdbfbab463750713118c8259a5d364547adab8ed91e94ba888349f33590a"},
+    {file = "opencv_python-4.5.4.58-cp39-cp39-win32.whl", hash = "sha256:9998ce60884f3cda074f02b56d2b57ee6bd863e2ddba132da2b0af3b9487d584"},
+    {file = "opencv_python-4.5.4.58-cp39-cp39-win_amd64.whl", hash = "sha256:5370a11757fbe94b176771269aff599f4da8676c2a672b13bcbca043f2e3eea8"},
+]
+pillow = [
+    {file = "Pillow-8.4.0-cp310-cp310-macosx_10_10_universal2.whl", hash = "sha256:81f8d5c81e483a9442d72d182e1fb6dcb9723f289a57e8030811bac9ea3fef8d"},
+    {file = "Pillow-8.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3f97cfb1e5a392d75dd8b9fd274d205404729923840ca94ca45a0af57e13dbe6"},
+    {file = "Pillow-8.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb9fc393f3c61f9054e1ed26e6fe912c7321af2f41ff49d3f83d05bacf22cc78"},
+    {file = "Pillow-8.4.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d82cdb63100ef5eedb8391732375e6d05993b765f72cb34311fab92103314649"},
+    {file = "Pillow-8.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:62cc1afda735a8d109007164714e73771b499768b9bb5afcbbee9d0ff374b43f"},
+    {file = "Pillow-8.4.0-cp310-cp310-win32.whl", hash = "sha256:e3dacecfbeec9a33e932f00c6cd7996e62f53ad46fbe677577394aaa90ee419a"},
+    {file = "Pillow-8.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:620582db2a85b2df5f8a82ddeb52116560d7e5e6b055095f04ad828d1b0baa39"},
+    {file = "Pillow-8.4.0-cp36-cp36m-macosx_10_10_x86_64.whl", hash = "sha256:1bc723b434fbc4ab50bb68e11e93ce5fb69866ad621e3c2c9bdb0cd70e345f55"},
+    {file = "Pillow-8.4.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:72cbcfd54df6caf85cc35264c77ede902452d6df41166010262374155947460c"},
+    {file = "Pillow-8.4.0-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:70ad9e5c6cb9b8487280a02c0ad8a51581dcbbe8484ce058477692a27c151c0a"},
+    {file = "Pillow-8.4.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25a49dc2e2f74e65efaa32b153527fc5ac98508d502fa46e74fa4fd678ed6645"},
+    {file = "Pillow-8.4.0-cp36-cp36m-win32.whl", hash = "sha256:93ce9e955cc95959df98505e4608ad98281fff037350d8c2671c9aa86bcf10a9"},
+    {file = "Pillow-8.4.0-cp36-cp36m-win_amd64.whl", hash = "sha256:2e4440b8f00f504ee4b53fe30f4e381aae30b0568193be305256b1462216feff"},
+    {file = "Pillow-8.4.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:8c803ac3c28bbc53763e6825746f05cc407b20e4a69d0122e526a582e3b5e153"},
+    {file = "Pillow-8.4.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c8a17b5d948f4ceeceb66384727dde11b240736fddeda54ca740b9b8b1556b29"},
+    {file = "Pillow-8.4.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1394a6ad5abc838c5cd8a92c5a07535648cdf6d09e8e2d6df916dfa9ea86ead8"},
+    {file = "Pillow-8.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:792e5c12376594bfcb986ebf3855aa4b7c225754e9a9521298e460e92fb4a488"},
+    {file = "Pillow-8.4.0-cp37-cp37m-win32.whl", hash = "sha256:d99ec152570e4196772e7a8e4ba5320d2d27bf22fdf11743dd882936ed64305b"},
+    {file = "Pillow-8.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:7b7017b61bbcdd7f6363aeceb881e23c46583739cb69a3ab39cb384f6ec82e5b"},
+    {file = "Pillow-8.4.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:d89363f02658e253dbd171f7c3716a5d340a24ee82d38aab9183f7fdf0cdca49"},
+    {file = "Pillow-8.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0a0956fdc5defc34462bb1c765ee88d933239f9a94bc37d132004775241a7585"},
+    {file = "Pillow-8.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b7bb9de00197fb4261825c15551adf7605cf14a80badf1761d61e59da347779"},
+    {file = "Pillow-8.4.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:72b9e656e340447f827885b8d7a15fc8c4e68d410dc2297ef6787eec0f0ea409"},
+    {file = "Pillow-8.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5a4532a12314149d8b4e4ad8ff09dde7427731fcfa5917ff16d0291f13609df"},
+    {file = "Pillow-8.4.0-cp38-cp38-win32.whl", hash = "sha256:82aafa8d5eb68c8463b6e9baeb4f19043bb31fefc03eb7b216b51e6a9981ae09"},
+    {file = "Pillow-8.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:066f3999cb3b070a95c3652712cffa1a748cd02d60ad7b4e485c3748a04d9d76"},
+    {file = "Pillow-8.4.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:5503c86916d27c2e101b7f71c2ae2cddba01a2cf55b8395b0255fd33fa4d1f1a"},
+    {file = "Pillow-8.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4acc0985ddf39d1bc969a9220b51d94ed51695d455c228d8ac29fcdb25810e6e"},
+    {file = "Pillow-8.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b052a619a8bfcf26bd8b3f48f45283f9e977890263e4571f2393ed8898d331b"},
+    {file = "Pillow-8.4.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:493cb4e415f44cd601fcec11c99836f707bb714ab03f5ed46ac25713baf0ff20"},
+    {file = "Pillow-8.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8831cb7332eda5dc89b21a7bce7ef6ad305548820595033a4b03cf3091235ed"},
+    {file = "Pillow-8.4.0-cp39-cp39-win32.whl", hash = "sha256:5e9ac5f66616b87d4da618a20ab0a38324dbe88d8a39b55be8964eb520021e02"},
+    {file = "Pillow-8.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:3eb1ce5f65908556c2d8685a8f0a6e989d887ec4057326f6c22b24e8a172c66b"},
+    {file = "Pillow-8.4.0-pp36-pypy36_pp73-macosx_10_10_x86_64.whl", hash = "sha256:ddc4d832a0f0b4c52fff973a0d44b6c99839a9d016fe4e6a1cb8f3eea96479c2"},
+    {file = "Pillow-8.4.0-pp36-pypy36_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9a3e5ddc44c14042f0844b8cf7d2cd455f6cc80fd7f5eefbe657292cf601d9ad"},
+    {file = "Pillow-8.4.0-pp36-pypy36_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c70e94281588ef053ae8998039610dbd71bc509e4acbc77ab59d7d2937b10698"},
+    {file = "Pillow-8.4.0-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:3862b7256046fcd950618ed22d1d60b842e3a40a48236a5498746f21189afbbc"},
+    {file = "Pillow-8.4.0-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a4901622493f88b1a29bd30ec1a2f683782e57c3c16a2dbc7f2595ba01f639df"},
+    {file = "Pillow-8.4.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84c471a734240653a0ec91dec0996696eea227eafe72a33bd06c92697728046b"},
+    {file = "Pillow-8.4.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:244cf3b97802c34c41905d22810846802a3329ddcb93ccc432870243211c79fc"},
+    {file = "Pillow-8.4.0.tar.gz", hash = "sha256:b8e2f83c56e141920c39464b852de3719dfbfb6e3c99a2d8da0edf4fb33176ed"},
+]
+protobuf = [
+    {file = "protobuf-3.19.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d80f80eb175bf5f1169139c2e0c5ada98b1c098e2b3c3736667f28cbbea39fc8"},
+    {file = "protobuf-3.19.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:a529e7df52204565bcd33738a7a5f288f3d2d37d86caa5d78c458fa5fabbd54d"},
+    {file = "protobuf-3.19.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28ccea56d4dc38d35cd70c43c2da2f40ac0be0a355ef882242e8586c6d66666f"},
+    {file = "protobuf-3.19.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8b30a7de128c46b5ecb343917d9fa737612a6e8280f440874e5cc2ba0d79b8f6"},
+    {file = "protobuf-3.19.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5935c8ce02e3d89c7900140a8a42b35bc037ec07a6aeb61cc108be8d3c9438a6"},
+    {file = "protobuf-3.19.1-cp36-cp36m-win32.whl", hash = "sha256:74f33edeb4f3b7ed13d567881da8e5a92a72b36495d57d696c2ea1ae0cfee80c"},
+    {file = "protobuf-3.19.1-cp36-cp36m-win_amd64.whl", hash = "sha256:038daf4fa38a7e818dd61f51f22588d61755160a98db087a046f80d66b855942"},
+    {file = "protobuf-3.19.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e51561d72efd5bd5c91490af1f13e32bcba8dab4643761eb7de3ce18e64a853"},
+    {file = "protobuf-3.19.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:6e8ea9173403219239cdfd8d946ed101f2ab6ecc025b0fda0c6c713c35c9981d"},
+    {file = "protobuf-3.19.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db3532d9f7a6ebbe2392041350437953b6d7a792de10e629c1e4f5a6b1fe1ac6"},
+    {file = "protobuf-3.19.1-cp37-cp37m-win32.whl", hash = "sha256:615b426a177780ce381ecd212edc1e0f70db8557ed72560b82096bd36b01bc04"},
+    {file = "protobuf-3.19.1-cp37-cp37m-win_amd64.whl", hash = "sha256:d8919368410110633717c406ab5c97e8df5ce93020cfcf3012834f28b1fab1ea"},
+    {file = "protobuf-3.19.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:71b0250b0cfb738442d60cab68abc166de43411f2a4f791d31378590bfb71bd7"},
+    {file = "protobuf-3.19.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:3cd0458870ea7d1c58e948ac8078f6ba8a7ecc44a57e03032ed066c5bb318089"},
+    {file = "protobuf-3.19.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:655264ed0d0efe47a523e2255fc1106a22f6faab7cc46cfe99b5bae085c2a13e"},
+    {file = "protobuf-3.19.1-cp38-cp38-win32.whl", hash = "sha256:b691d996c6d0984947c4cf8b7ae2fe372d99b32821d0584f0b90277aa36982d3"},
+    {file = "protobuf-3.19.1-cp38-cp38-win_amd64.whl", hash = "sha256:e7e8d2c20921f8da0dea277dfefc6abac05903ceac8e72839b2da519db69206b"},
+    {file = "protobuf-3.19.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fd390367fc211cc0ffcf3a9e149dfeca78fecc62adb911371db0cec5c8b7472d"},
+    {file = "protobuf-3.19.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d83e1ef8cb74009bebee3e61cc84b1c9cd04935b72bca0cbc83217d140424995"},
+    {file = "protobuf-3.19.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36d90676d6f426718463fe382ec6274909337ca6319d375eebd2044e6c6ac560"},
+    {file = "protobuf-3.19.1-cp39-cp39-win32.whl", hash = "sha256:e7b24c11df36ee8e0c085e5b0dc560289e4b58804746fb487287dda51410f1e2"},
+    {file = "protobuf-3.19.1-cp39-cp39-win_amd64.whl", hash = "sha256:77d2fadcf369b3f22859ab25bd12bb8e98fb11e05d9ff9b7cd45b711c719c002"},
+    {file = "protobuf-3.19.1-py2.py3-none-any.whl", hash = "sha256:e813b1c9006b6399308e917ac5d298f345d95bb31f46f02b60cd92970a9afa17"},
+    {file = "protobuf-3.19.1.tar.gz", hash = "sha256:62a8e4baa9cb9e064eb62d1002eca820857ab2138440cb4b3ea4243830f94ca7"},
+]

pyproject.toml

@@ -0,0 +1,19 @@
+[tool.poetry]
+name = "openseeface"
+version = "1.20.4"
+description = ""
+authors = ["emilianavt"]
+license = "BSD-2-Clause"
+
+[tool.poetry.dependencies]
+python = ">=3.7,<3.11"
+numpy = "^1.21.3"
+opencv-python = "^4.5.4"
+Pillow = "^8.4.0"
+onnxruntime = "^1.9.0"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

retinaface.py

@@ -66,7 +66,8 @@ class RetinaFaceDetector():
         options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
         options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
         options.log_severity_level = 3
-        self.session = onnxruntime.InferenceSession(model_path, sess_options=options)
+        providersList = onnxruntime.capi._pybind_state.get_available_providers()
+        self.session = onnxruntime.InferenceSession(model_path, sess_options=options, providers=providersList)
         self.res_w, self.res_h = res
         with open(json_path, "r") as prior_file:
             self.priorbox = np.array(json.loads(prior_file.read()))

run.bat

@@ -2,12 +2,15 @@
 
 facetracker -l 1
 
+echo Make sure that nothing is accessing your camera before you proceed.
 
 set /p cameraNum=Select your camera from the list above and enter the corresponding number:
-set /p width=Select the width:
+
-set /p height=Select the height:
+facetracker -a %cameraNum%
+
+set /p dcaps=Select your camera mode or -1 for default settings:
 set /p fps=Select the FPS:
 
-facetracker -c %cameraNum% -W %width% -H %height% -F %fps% -v 3 -P 1 --discard-after 0 --scan-every 0 --no-3d-adapt 1 --max-feature-updates 900
+facetracker -c %cameraNum% -F %fps% -D %dcaps% -v 3 -P 1 --discard-after 0 --scan-every 0 --no-3d-adapt 1 --max-feature-updates 900
 
 pause

tracker.py

@@ -50,7 +50,7 @@ def compensate(p1, p2):
 def rotate_image(image, a, center):
     (h, w) = image.shape[:2]
     a = np.rad2deg(a)
-    M = cv2.getRotationMatrix2D((center[0], center[1]), a, 1.0)
+    M = cv2.getRotationMatrix2D((float(center[0]), float(center[1])), a, 1.0)
     rotated = cv2.warpAffine(image, M, (w, h))
     return rotated
 
@@ -140,7 +140,7 @@ def worker_thread(session, frame, input, crop_info, queue, input_name, idx, trac
     conf, lms = tracker.landmarks(output[0], crop_info)
     if conf > tracker.threshold:
         try:
-            eye_state = tracker.get_eye_state(frame, lms, single=True)
+            eye_state = tracker.get_eye_state(frame, lms)
         except:
             eye_state = [(1.0, 0.0, 0.0, 0.0), (1.0, 0.0, 0.0, 0.0)]
         queue.put((session, conf, (lms, eye_state), crop_info, idx))
@@ -263,11 +263,11 @@ class FeatureExtractor():
         norm_distance_y = np.mean([pts[27, 1] - pts[28, 1], pts[28, 1] - pts[29, 1], pts[29, 1] - pts[30, 1]])
 
         a1, f_pts = self.align_points(pts[42], pts[45], pts[[43, 44, 47, 46]])
-        f = np.clip((np.mean([f_pts[0,1], f_pts[1,1]]) - np.mean([f_pts[2,1], f_pts[3,1]])) / norm_distance_y, 0, None)
+        f = abs((np.mean([f_pts[0,1], f_pts[1,1]]) - np.mean([f_pts[2,1], f_pts[3,1]])) / norm_distance_y)
         features["eye_l"] = self.eye_l.update(f, now)
 
         a2, f_pts = self.align_points(pts[36], pts[39], pts[[37, 38, 41, 40]])
-        f = np.clip((np.mean([f_pts[0,1], f_pts[1,1]]) - np.mean([f_pts[2,1], f_pts[3,1]])) / norm_distance_y, 0, None)
+        f = abs((np.mean([f_pts[0,1], f_pts[1,1]]) - np.mean([f_pts[2,1], f_pts[3,1]])) / norm_distance_y)
         features["eye_r"] = self.eye_r.update(f, now)
 
         if full:
@@ -315,7 +315,7 @@ class FeatureExtractor():
         else:
             features["mouth_corner_inout_r"] = 0.
 
-        f = (np.mean([pts[59, 1], pts[60, 1], pts[61, 1]]) - np.mean([pts[65, 1], pts[64, 1], pts[63, 1]])) / norm_distance_y
+        f = abs(np.mean(pts[[59,60,61], 1], axis=0) - np.mean(pts[[63,64,65], 1], axis=0)) / norm_distance_y
         features["mouth_open"] = self.mouth_open.update(f, now)
 
         f = abs(pts[58, 0] - pts[62, 0]) / norm_distance_x
@@ -328,9 +328,9 @@ class FaceInfo():
         self.id = id
         self.frame_count = -1
         self.tracker = tracker
-        self.contour_pts = [0,1,8,15,16,27,28,29,30,31,32,33,34,35,36,39,42,45]
+        self.contour_pts = [0,1,8,15,16,27,28,29,30,31,32,33,34,35]
         self.face_3d = copy.copy(self.tracker.face_3d)
-        if self.tracker.model_type < 0:
+        if self.tracker.model_type == -1:
             self.contour_pts = [0,2,8,14,16,27,30,33]
         self.reset()
         self.alive = False
@@ -401,7 +401,7 @@ class FaceInfo():
         if self.conf < 0.4 or self.pnp_error > 300:
             return
 
-        if self.tracker.model_type > -1 and not self.tracker.static_model:
+        if self.tracker.model_type != -1 and not self.tracker.static_model:
             max_runs = 1
             eligible = np.delete(np.arange(0, 66), [30])
             changed_any = False
@@ -484,11 +484,20 @@ class FaceInfo():
             self.eye_blink.append(1 - min(max(0, -self.current_features["eye_r"]), 1))
             self.eye_blink.append(1 - min(max(0, -self.current_features["eye_l"]), 1))
 
+def get_model_base_path(model_dir):
+    model_base_path = resolve(os.path.join("models"))
+    if model_dir is None:
+        if not os.path.exists(model_base_path):
+            model_base_path = resolve(os.path.join("..", "models"))
+    else:
+        model_base_path = model_dir
+    return model_base_path
+
 class Tracker():
-    def __init__(self, width, height, model_type=3, detection_threshold=0.6, threshold=None, max_faces=1, discard_after=5, scan_every=3, bbox_growth=0.0, max_threads=4, silent=False, model_dir=None, no_gaze=False, use_retinaface=False, max_feature_updates=0, static_model=False, feature_level=2):
+    def __init__(self, width, height, model_type=3, detection_threshold=0.6, threshold=None, max_faces=1, discard_after=5, scan_every=3, bbox_growth=0.0, max_threads=4, silent=False, model_dir=None, no_gaze=False, use_retinaface=False, max_feature_updates=0, static_model=False, feature_level=2, try_hard=False):
         options = onnxruntime.SessionOptions()
         options.inter_op_num_threads = 1
-        options.intra_op_num_threads = max(max_threads,4)
+        options.intra_op_num_threads = min(max_threads,4)
         options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
         options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
         options.log_severity_level = 3
@@ -497,17 +506,17 @@ class Tracker():
             "lm_model0_opt.onnx",
             "lm_model1_opt.onnx",
             "lm_model2_opt.onnx",
-            "lm_model3_opt.onnx"
+            "lm_model3_opt.onnx",
+            "lm_model4_opt.onnx"
         ]
         model = "lm_modelT_opt.onnx"
         if model_type >= 0:
             model = self.models[self.model_type]
-        model_base_path = resolve(os.path.join("models"))
+        if model_type == -2:
-        if model_dir is None:
+            model = "lm_modelV_opt.onnx"
-            if not os.path.exists(model_base_path):
+        if model_type == -3:
-                model_base_path = resolve(os.path.join("..", "models"))
+            model = "lm_modelU_opt.onnx"
-        else:
+        model_base_path = get_model_base_path(model_dir)
-            model_base_path = model_dir
 
         if threshold is None:
             threshold = 0.6
@@ -518,37 +527,39 @@ class Tracker():
         self.retinaface_scan = RetinaFaceDetector(model_path=os.path.join(model_base_path, "retinaface_640x640_opt.onnx"), json_path=os.path.join(model_base_path, "priorbox_640x640.json"), threads=2, top_k=max_faces, res=(640, 640))
         self.use_retinaface = use_retinaface
 
+        # OTR 1.9 and later requires specifying providers
+        # explicitly as an argument in InferenceSession()
+        providersList = onnxruntime.capi._pybind_state.get_available_providers()
+
         # Single face instance with multiple threads
-        self.session = onnxruntime.InferenceSession(os.path.join(model_base_path, model), sess_options=options)
+        self.session = onnxruntime.InferenceSession(os.path.join(model_base_path, model), sess_options=options, providers=providersList)
 
         # Multiple faces with single threads
         self.sessions = []
-        self.max_workers = min(max_threads, max_faces)
+        self.max_workers = max(min(max_threads, max_faces), 1)
         extra_threads = max_threads % self.max_workers
         for i in range(self.max_workers):
             options = onnxruntime.SessionOptions()
             options.inter_op_num_threads = 1
-            options.intra_op_num_threads = max_threads // self.max_workers
+            options.intra_op_num_threads = min(max(max_threads // self.max_workers, 4), 1)
             if options.intra_op_num_threads < 1:
                 options.intra_op_num_threads = 1
             elif i < extra_threads:
                 options.intra_op_num_threads += 1
             options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
             options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
-            self.sessions.append(onnxruntime.InferenceSession(os.path.join(model_base_path, model), sess_options=options))
+            self.sessions.append(onnxruntime.InferenceSession(os.path.join(model_base_path, model), sess_options=options, providers=providersList))
         self.input_name = self.session.get_inputs()[0].name
 
         options = onnxruntime.SessionOptions()
         options.inter_op_num_threads = 1
-        options.intra_op_num_threads = max(max_threads,4)
+        options.intra_op_num_threads = 1
         options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
         options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
         options.log_severity_level = 3
-        self.gaze_model = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options)
+        self.gaze_model = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options, providers=providersList)
-        options.intra_op_num_threads = 1
-        self.gaze_model_single = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_split_opt.onnx"), sess_options=options)
 
-        self.detection = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_detection_opt.onnx"), sess_options=options)
+        self.detection = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_detection_opt.onnx"), sess_options=options, providers=providersList)
         self.faces = []
 
         # Image normalization constants
@@ -657,6 +668,7 @@ class Tracker():
         self.scan_every = scan_every
         self.bbox_growth = bbox_growth
         self.silent = silent
+        self.try_hard = try_hard
 
         self.res = 224.
         self.mean_res = self.mean_224
@@ -665,19 +677,27 @@ class Tracker():
             self.res = 56.
             self.mean_res = np.tile(self.mean, [56, 56, 1])
             self.std_res = np.tile(self.std, [56, 56, 1])
+        if model_type < -1:
+            self.res = 112.
+            self.mean_res = np.tile(self.mean, [112, 112, 1])
+            self.std_res = np.tile(self.std, [112, 112, 1])
         self.res_i = int(self.res)
         self.out_res = 27.
         if model_type < 0:
             self.out_res = 6.
+        if model_type < -1:
+            self.out_res = 13.
         self.out_res_i = int(self.out_res) + 1
         self.logit_factor = 16.
         if model_type < 0:
             self.logit_factor = 8.
+        if model_type < -1:
+            self.logit_factor = 16.
 
         self.no_gaze = no_gaze
         self.debug_gaze = False
         self.feature_level = feature_level
-        if model_type < 0:
+        if model_type == -1:
             self.feature_level = min(feature_level, 1)
         self.max_feature_updates = max_feature_updates
         self.static_model = static_model
@@ -715,7 +735,7 @@ class Tracker():
         avg_conf = 0
         res = self.res - 1
         c0, c1, c2 = 66, 132, 198
-        if self.model_type < 0:
+        if self.model_type == -1:
             c0, c1, c2 = 30, 60, 90
         t_main = tensor[0:c0].reshape((c0,self.out_res_i * self.out_res_i))
         t_m = t_main.argmax(1)
@@ -730,7 +750,7 @@ class Tracker():
         avg_conf = np.average(t_conf)
         lms = np.stack([t_x, t_y, t_conf], 1)
         lms[np.isnan(lms).any(axis=1)] = np.array([0.,0.,0.], dtype=np.float32)
-        if self.model_type < 0:
+        if self.model_type == -1:
             lms = lms[[0,0,1,1,1,2,2,2,3,3,3,4,4,4,5,5,6,7,7,8,8,9,10,10,11,11,12,21,21,21,22,23,23,23,23,23,13,14,14,15,16,16,17,18,18,19,20,20,24,25,25,25,26,26,27,27,27,24,24,28,28,28,26,29,29,29]]
             #lms[[1,3,4,6,7,9,10,12,13,15,18,20,23,25,38,40,44,46]] += lms[[2,2,5,5,8,8,11,11,14,16,19,21,24,26,39,39,45,45]]
             #lms[[3,4,6,7,9,10,12,13]] += lms[[5,5,8,8,11,11,14,14]]
@@ -742,12 +762,12 @@ class Tracker():
         return (avg_conf, np.array(lms))
 
     def estimate_depth(self, face_info):
-        lms = np.concatenate((face_info.lms, np.array([[face_info.eye_state[0][1], face_info.eye_state[0][2], face_info.eye_state[0][3]], [face_info.eye_state[1][1], face_info.eye_state[1][2], face_info.eye_state[1][3]]], np.float)), 0)
+        lms = np.concatenate((face_info.lms, np.array([[face_info.eye_state[0][1], face_info.eye_state[0][2], face_info.eye_state[0][3]], [face_info.eye_state[1][1], face_info.eye_state[1][2], face_info.eye_state[1][3]]], np.float32)), 0)
 
         image_pts = np.array(lms)[face_info.contour_pts, 0:2]
 
         success = False
-        if not face_info.rotation is None:
+        if face_info.rotation is not None:
             success, face_info.rotation, face_info.translation = cv2.solvePnP(face_info.contour, image_pts, self.camera, self.dist_coeffs, useExtrinsicGuess=True, rvec=np.transpose(face_info.rotation), tvec=np.transpose(face_info.translation), flags=cv2.SOLVEPNP_ITERATIVE)
         else:
             rvec = np.array([0, 0, 0], np.float32)
@@ -763,8 +783,10 @@ class Tracker():
             face_info.translation = np.array([0.0, 0.0, 0.0], np.float32)
             return False, np.zeros(4), np.zeros(3), 99999., pts_3d, lms
         else:
-            face_info.rotation = np.transpose(face_info.rotation)
+            # OpenCV returns (3,1); flatten so [0]/[1]/[2] are scalars for UDP packing
-            face_info.translation = np.transpose(face_info.translation)
+            # and 1D .dot() math does not broadcast to (1,3).
+            face_info.rotation = np.asarray(face_info.rotation, dtype=np.float32).reshape(3)
+            face_info.translation = np.asarray(face_info.translation, dtype=np.float32).reshape(3)
 
         rmat, _ = cv2.Rodrigues(rotation)
         inverse_rotation = np.linalg.inv(rmat)
@@ -813,7 +835,7 @@ class Tracker():
                 reference = rmat.dot(pt)
                 reference = reference + face_info.translation
                 reference = self.camera.dot(reference)
-                depth = reference[2]
+                depth = float(reference[2])
                 pt_3d = np.array([lms[66+i][0] * depth, lms[66+i][1] * depth, depth], np.float32)
                 pt_3d = self.inverse_camera.dot(pt_3d)
                 pt_3d = pt_3d - face_info.translation
@@ -900,11 +922,11 @@ class Tracker():
         x1, y1 = clamp_to_im(center - radius, h, w)
         x2, y2 = clamp_to_im(center + radius + 1, h, w)
         offset = np.array((x1, y1))
-        lms = (lms[:, 0:2] - offset).astype(np.int)
+        lms = (lms[:, 0:2] - offset).astype(int)
         frame = frame[y1:y2, x1:x2]
         return frame, lms, offset
 
-    def get_eye_state(self, frame, lms, single=False):
+    def get_eye_state(self, frame, lms):
         if self.no_gaze:
             return [(1.0, 0.0, 0.0, 0.0), (1.0, 0.0, 0.0, 0.0)]
         lms = np.array(lms)
@@ -920,10 +942,7 @@ class Tracker():
             return [(1.0, 0.0, 0.0, 0.0), (1.0, 0.0, 0.0, 0.0)]
         both_eyes = np.concatenate((right_eye, left_eye))
         results = None
-        if single:
+        results = self.gaze_model.run([], {self.input_name: both_eyes})
-            results = self.gaze_model_single.run([], {self.input_name: both_eyes})
-        else:
-            results = self.gaze_model.run([], {self.input_name: both_eyes})
         open = [0, 0]
         open[0] = 1#results[1][0].argmax()
         open[1] = 1#results[1][1].argmax()
@@ -1030,11 +1049,13 @@ class Tracker():
         self.wait_count += 1
         if self.detected == 0:
             start_fd = time.perf_counter()
-            if self.use_retinaface > 0:
+            if self.use_retinaface > 0 or self.try_hard:
                 retinaface_detections = self.retinaface.detect_retina(frame)
                 new_faces.extend(retinaface_detections)
-            else:
+            if self.use_retinaface == 0 or self.try_hard:
                 new_faces.extend(self.detect_faces(frame))
+            if self.try_hard:
+                new_faces.extend([(0, 0, self.width, self.height)])
             duration_fd = 1000 * (time.perf_counter() - start_fd)
             self.wait_count = 0
         elif self.detected < self.max_faces:
@@ -1185,7 +1206,7 @@ class Tracker():
 
         duration = (time.perf_counter() - start) * 1000
         if not self.silent:
-            print(f"Took {duration:.2f}ms (detect: {duration_fd:.2f}ms, crop: {duration_pp:.2f}, track: {duration_model:.2f}ms, 3D points: {duration_pnp:.2f}ms)")
+            print(f"Took {duration:.2f}ms (detect: {duration_fd:.2f}ms, crop: {duration_pp:.2f}ms, track: {duration_model:.2f}ms, 3D points: {duration_pnp:.2f}ms)")
 
         results = sorted(results, key=lambda x: x.id)