Fix PnP rotation/translation shape for modern OpenCV/NumPy.

solvePnP returns (3,1); transpose made (1,3), which broke eye depth .dot()
and struct.pack of translation scalars. Flatten to (3,) instead.

Co-authored-by: Cursor <cursoragent@cursor.com>

.github/actions/upload-dated-artifact/action.yaml

@@ -0,0 +1,26 @@
+name: Upload dated artifact
+description: Uploads an artifact with a datetime
+
+inputs:
+  name:
+    description: The name of the artifact
+    required: true
+  path:
+    description: The path to the file/folder to upload
+    required: true
+
+runs:
+  using: composite
+  steps:
+    # https://stackoverflow.com/questions/60942067/get-current-date-and-time-in-github-workflows
+    - name: Get current datetime
+      id: date
+      shell: bash
+      run: echo "::set-output name=date::$(date '+%Y-%m-%d_%H-%M-%S')"
+
+    - name: Upload artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: ${{ inputs.name }}_${{ steps.date.outputs.date }}
+        path: ${{ inputs.path }}
+        if-no-files-found: error

.github/workflows/create-binaries.yaml

@@ -0,0 +1,87 @@
+name: Package using PyInstaller
+on:
+  push:
+    tags:
+      - v**
+  workflow_dispatch:
+
+jobs:
+  build-windows:
+    env:
+      dist-path: dist/OpenSeeFace
+    runs-on: windows-latest
+    steps:
+      - name: Pull OpenSeeFace
+        uses: actions/checkout@v3
+
+      - name: Create binary
+        shell: cmd
+        run: |
+          choco install vcredist140
+
+          xcopy c:\Windows\System32\msvcp140.dll .
+          xcopy c:\Windows\System32\vcomp140.dll .
+          xcopy c:\Windows\System32\concrt140.dll .
+          xcopy c:\Windows\System32\vccorlib140.dll .
+
+          make_exe.bat
+
+      - name: Move files for uploading
+        shell: bash
+        run: |
+          # Prevents the parent dist/ directory from being included
+          mv dist/facetracker ${{ env.dist-path }}
+          cp -r models ${{ env.dist-path }}
+
+      - name: Upload artifact
+        uses: ./.github/actions/upload-dated-artifact
+        with:
+          name: windows
+          path: dist # Add the parent directory to prevent a tarbomb
+
+  # Files must be packed into a tar in order for file permissions to be preserved
+  build-unix:
+    env:
+      artifact-path: OpenSeeFace
+      tar-path: OpenSeeFace.tar
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: linux
+            os: ubuntu-latest
+          - name: mac
+            os: macos-latest
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Pull OpenSeeFace
+        uses: actions/checkout@v3
+
+      - name: Create binary
+        shell: bash
+        run: |
+          chmod +x make_exe.sh
+          ./make_exe.sh
+
+      - name: Move files for uploading
+        shell: bash
+        run: |
+          # Prevents the parent dist/ directory from being included
+          mv dist/facetracker ${{ env.artifact-path }}
+          cp -r models ${{ env.artifact-path }}
+
+      - name: Fix permissions from PyInstaller
+        shell: bash
+        run: |
+          find ${{ env.artifact-path }} -type f -exec chmod -x {} +
+          chmod +x ${{ env.artifact-path }}/facetracker
+
+      - name: Pack files into tar
+        shell: bash
+        run: tar -cvf ${{ env.tar-path }} ${{ env.artifact-path }}
+
+      - name: Upload artifact
+        uses: ./.github/actions/upload-dated-artifact
+        with:
+          name: ${{ matrix.name }}
+          path: ${{ env.tar-path }}

.gitignore

@@ -4,4 +4,5 @@ Binary/*
 Samples/*
 *.zip
 *.7z
-CSharp/*
+CSharp/*
+VSeeFace/*

Examples/Animations/JawOpen.anim.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 697b139e69dcb414498dcd93210620ff
+NativeFormatImporter:
+  externalObjects: {}
+  mainObjectFileID: 0
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Examples/OpenSeeVRMDriver.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 735f1f2cba9bb364892bc2c7e7a30c19
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Examples/OpenSeeVRMExpression.cs

@@ -0,0 +1,78 @@
+using System.Collections.Generic;
+using UnityEngine;
+using VRM;
+
+namespace OpenSee {
+
+public class OpenSeeVRMExpression : MonoBehaviour
+{
+    public OpenSee openSee;
+    public OpenSeeExpression openSeeExpression;
+    public VRMBlendShapeProxy vrmBlendShapeProxy;
+    public string expressionNeutral = "neutral";
+    public string expressionFun = "smile";
+    public string expressionSorrow = "sad";
+    public string expressionAngry = "angry";
+    public string expressionJoy = "shocked";
+    
+    private BlendShapeKey lastKey;
+    
+    void Start()
+    {
+    }
+    
+    void Update()
+    {
+        if (openSeeExpression == null || vrmBlendShapeProxy == null) {
+            return;
+        }
+        float rightEyeOpen = 1f;
+        float leftEyeOpen = 1f;
+        if (openSee != null) {
+            var openSeeData = openSee.trackingData;
+            if (openSeeData == null || openSeeData.Length < 1)
+                return;
+            int idx = -1;
+            int i = 0;
+            foreach (var item in openSeeData) {
+                if (item.id == openSeeExpression.faceId)
+                    idx = i;
+                i++;
+            }
+            if (idx > -1) {
+                rightEyeOpen = openSeeData[idx].rightEyeOpen;
+                leftEyeOpen = openSeeData[idx].leftEyeOpen;
+            }
+        }
+        vrmBlendShapeProxy.ImmediatelySetValue(BlendShapeKey.CreateFromPreset(BlendShapePreset.Neutral), 0f);
+        vrmBlendShapeProxy.ImmediatelySetValue(BlendShapeKey.CreateFromPreset(BlendShapePreset.Fun), 0f);
+        vrmBlendShapeProxy.ImmediatelySetValue(BlendShapeKey.CreateFromPreset(BlendShapePreset.Sorrow), 0f);
+        vrmBlendShapeProxy.ImmediatelySetValue(BlendShapeKey.CreateFromPreset(BlendShapePreset.Angry), 0f);
+        vrmBlendShapeProxy.ImmediatelySetValue(BlendShapeKey.CreateFromPreset(BlendShapePreset.Joy), 0f);
+        vrmBlendShapeProxy.ImmediatelySetValue(BlendShapeKey.CreateFromPreset(BlendShapePreset.Blink_R), 0f);
+        vrmBlendShapeProxy.ImmediatelySetValue(BlendShapeKey.CreateFromPreset(BlendShapePreset.Blink_L), 0f);
+        if (openSeeExpression.expression == expressionFun) {
+            vrmBlendShapeProxy.ImmediatelySetValue(BlendShapeKey.CreateFromPreset(BlendShapePreset.Fun), 1f);
+            return;
+        } else if (openSeeExpression.expression == expressionSorrow) {
+            vrmBlendShapeProxy.ImmediatelySetValue(BlendShapeKey.CreateFromPreset(BlendShapePreset.Sorrow), 1f);
+            return;
+        } else if (openSeeExpression.expression == expressionAngry) {
+            vrmBlendShapeProxy.ImmediatelySetValue(BlendShapeKey.CreateFromPreset(BlendShapePreset.Angry), 1f);
+            return;
+        } else if (openSeeExpression.expression == expressionJoy) {
+            vrmBlendShapeProxy.ImmediatelySetValue(BlendShapeKey.CreateFromPreset(BlendShapePreset.Joy), 1f);
+            return;
+        } else {
+            /*if (rightEyeOpen < 0.05) {
+                vrmBlendShapeProxy.ImmediatelySetValue(new BlendShapeKey(BlendShapePreset.Blink_R), 1f);
+            }
+            if (leftEyeOpen < 0.05) {
+                vrmBlendShapeProxy.ImmediatelySetValue(new BlendShapeKey(BlendShapePreset.Blink_L), 1f);
+            }*/
+        }
+        vrmBlendShapeProxy.ImmediatelySetValue(BlendShapeKey.CreateFromPreset(BlendShapePreset.Neutral), 1f);
+    }
+}
+
+}

Examples/OpenSeeVRMExpression.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 3c7910e1fb62bf449920cea2fdf07326
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Examples/OpenSeeWebcamInfo.cs

@@ -0,0 +1,270 @@
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Net;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+
+namespace OpenSee {
+
+[Serializable]
+public enum OpenSeeWebcamType { Unknown = -1, DirectShow, Blackmagic };
+
+[Serializable]
+public enum OpenSeeWebcamFormat { Any = 0, Unknown = 1, ARGB = 100, XRGB, I420 = 200, NV12, YV12, Y800, YVYU = 300, YUY2, UYVY, HDYC, MJPEG = 400, H264 };
+
+[Serializable]
+public struct OpenSeeWebcamCapability {
+    public int id;
+    
+    // DirectShow
+    public int minCX;
+    public int minCY;
+    public int maxCX;
+    public int maxCY;
+    public int granularityCX;
+    public int granularityCY;
+    public int minInterval;
+    public int maxInterval;
+    public int rating;
+    public OpenSeeWebcamFormat format;
+    
+    // Blackmagic
+    public int bmTimescale;
+    public int bmFrameduration;
+    public int bmModecode;
+}
+
+[Serializable]
+public class OpenSeeWebcam {
+    public OpenSeeWebcamType type = OpenSeeWebcamType.Unknown;
+    public int id;
+    public string name;
+
+    // DirectShow
+    public string path;
+    
+    // Blackmagic
+    public long bmId;
+    public int bmFlags;
+
+    // General
+    public OpenSeeWebcamCapability[] caps;
+    
+    public List<string> prettyCaps = null;
+    
+    private List<Tuple<OpenSeeWebcamCapability, int>> splitCaps = null;
+    
+    bool ge(bool a, bool b) {
+        return a || (!a && !b);
+    }
+
+    private int CompareCaps(OpenSeeWebcamCapability a, OpenSeeWebcamCapability b) {
+        int aRes = a.minCX * a.minCY;
+        int bRes = b.minCX * b.minCY;
+        bool aGoodFps = a.minInterval <= 670000;
+        bool bGoodFps = a.minInterval <= 670000;
+        bool aGoodRes = aRes >= 850000 && aRes <= 2200000;
+        bool bGoodRes = bRes >= 850000 && bRes <= 2200000;
+        if (a.rating < b.rating && ge(aGoodFps, bGoodFps) && ge(aGoodRes, bGoodRes))
+            return -1;
+        if (a.rating > b.rating && ge(bGoodFps, aGoodFps) && ge(bGoodRes, aGoodRes))
+            return 1;
+        if (aRes > bRes && ge(aGoodFps, bGoodFps) && ge(aGoodRes, bGoodRes))
+            return -1;
+        if (aRes < bRes && ge(bGoodFps, aGoodFps) && ge(bGoodRes, aGoodRes))
+            return 1;
+        if (a.minInterval < b.minInterval)
+            return -1;
+        if (a.minInterval > b.minInterval)
+            return 1;
+        return 0;
+    }
+    
+    private string GetPrettyCapability(OpenSeeWebcamCapability cap) {
+        float fps = 10000000f / (float)cap.minInterval;
+        string prettyCap = cap.minCX + "x" + cap.minCY + ", " + fps.ToString("0.##") + "fps (" + cap.format.ToString() + ")";
+        return prettyCap;
+    }
+    
+    public List<string> GetPrettyCapabilities() {
+        if (splitCaps != null && prettyCaps != null)
+            return prettyCaps;
+        
+        splitCaps = new List<Tuple<OpenSeeWebcamCapability, int>>();
+        
+        for (int i = 0; i < caps.Length; i++) {
+            if (caps[i].minCX == caps[i].maxCX && caps[i].minCY == caps[i].maxCY)
+                splitCaps.Add(new Tuple<OpenSeeWebcamCapability, int>(caps[i], i));
+            else {
+                OpenSeeWebcamCapability min = caps[i];
+                OpenSeeWebcamCapability max = caps[i];
+                min.maxCX = min.minCX;
+                min.maxCY = min.minCY;
+                max.minCX = max.maxCX;
+                max.minCY = max.maxCY;
+                splitCaps.Add(new Tuple<OpenSeeWebcamCapability, int>(min, i));
+                splitCaps.Add(new Tuple<OpenSeeWebcamCapability, int>(max, i));
+            }
+        }
+        
+        splitCaps.Sort((a, b) => CompareCaps(a.Item1, b.Item1));
+        
+        prettyCaps = new List<string>();
+        if (type == OpenSeeWebcamType.DirectShow)
+            prettyCaps.Add("Default settings");
+        foreach (var cap in splitCaps) {
+            prettyCaps.Add(GetPrettyCapability(cap.Item1));
+        }
+        
+        return prettyCaps;
+    }
+    
+    public OpenSeeWebcamCapability GetCapabilityByPrettyIndex(int index) {
+        int dshowOffset = 0;
+        if (type == OpenSeeWebcamType.DirectShow) {
+            if (index == 0) {
+                OpenSeeWebcamCapability cap = new OpenSeeWebcamCapability();
+                cap.id = -1;
+                cap.format = OpenSeeWebcamFormat.Any;
+                cap.minInterval = 10000000 / 60;
+                return cap;
+            }
+            dshowOffset = 1;
+        }
+        
+        if (splitCaps == null || index >= splitCaps.Count + dshowOffset || index < 0)
+            throw new InvalidOperationException("Invalid capability index.");
+        
+        return caps[splitCaps[index - dshowOffset].Item2];
+    }
+}
+
+[DefaultExecutionOrder(-50)]
+public class OpenSeeWebcamInfo : MonoBehaviour {
+    #region DllImport
+    // DirectShow
+    [DllImport("dshowcapture_x64", CallingConvention = CallingConvention.Cdecl, EntryPoint = "create_capture")]
+	private static extern System.IntPtr create_capture_x64();
+
+    [DllImport("dshowcapture_x64", CallingConvention = CallingConvention.Cdecl, EntryPoint = "get_json_length")]
+	private static extern int get_json_length_x64(System.IntPtr cap);
+
+    [DllImport("dshowcapture_x64", CallingConvention = CallingConvention.Cdecl, EntryPoint = "get_json")]
+	private static extern void get_json_x64(System.IntPtr cap, [Out] StringBuilder namebuffer, int bufferlength);
+    
+    [DllImport("dshowcapture_x64", CallingConvention = CallingConvention.Cdecl, EntryPoint = "destroy_capture")]
+	private static extern void destroy_capture_x64(System.IntPtr cap);
+
+    [DllImport("dshowcapture_x86", CallingConvention = CallingConvention.Cdecl, EntryPoint = "create_capture")]
+	private static extern System.IntPtr create_capture_x86();
+
+    [DllImport("dshowcapture_x86", CallingConvention = CallingConvention.Cdecl, EntryPoint = "get_json_length")]
+	private static extern int get_json_length_x86(System.IntPtr cap);
+
+    [DllImport("dshowcapture_x86", CallingConvention = CallingConvention.Cdecl, EntryPoint = "get_json")]
+	private static extern void get_json_x86(System.IntPtr cap, [Out] StringBuilder namebuffer, int bufferlength);
+
+    [DllImport("dshowcapture_x86", CallingConvention = CallingConvention.Cdecl, EntryPoint = "destroy_capture")]
+	private static extern void destroy_capture_x86(System.IntPtr cap);
+
+    // Blackmagic
+    [DllImport("libminibmcapture64", CallingConvention = CallingConvention.Cdecl, EntryPoint = "get_json_length")]
+	private static extern int bm_get_json_length_x64();
+
+    [DllImport("libminibmcapture64", CallingConvention = CallingConvention.Cdecl, EntryPoint = "get_json")]
+	private static extern void bm_get_json_x64([Out] StringBuilder namebuffer, int bufferlength);
+
+    [DllImport("libminibmcapture32", CallingConvention = CallingConvention.Cdecl, EntryPoint = "get_json_length")]
+	private static extern int bm_get_json_length_x86();
+
+    [DllImport("libminibmcapture32", CallingConvention = CallingConvention.Cdecl, EntryPoint = "get_json")]
+	private static extern void bm_get_json_x86([Out] StringBuilder namebuffer, int bufferlength);
+    #endregion
+    
+    [Serializable]
+    public class OpenSeeWebcamList {
+        public List<OpenSeeWebcam> list;
+    }
+
+    public List<OpenSeeWebcam> cameras;
+    public bool dumpJson = false;
+    
+    private static bool dumpJsonStatic = false;
+    
+    static private string ListCameraDetails_x64() {
+        System.IntPtr cap = create_capture_x64();
+		int length = get_json_length_x64(cap);
+        StringBuilder buffer = new StringBuilder(length);
+        get_json_x64(cap, buffer, length);
+        destroy_capture_x64(cap);
+		return buffer.ToString();
+    }
+
+    static private string ListCameraDetails_x86() {
+        System.IntPtr cap = create_capture_x86();
+		int length = get_json_length_x86(cap);
+        StringBuilder buffer = new StringBuilder(length);
+        get_json_x86(cap, buffer, length);
+        destroy_capture_x86(cap);
+		return buffer.ToString();
+    }
+
+    static private string ListBlackMagicDetails_x64() {
+		int length = bm_get_json_length_x64();
+        StringBuilder buffer = new StringBuilder(length);
+        bm_get_json_x64(buffer, length);
+		return buffer.ToString();
+    }
+
+    static private string ListBlackMagicDetails_x86() {
+		int length = bm_get_json_length_x86();
+        StringBuilder buffer = new StringBuilder(length);
+        bm_get_json_x86(buffer, length);
+		return buffer.ToString();
+    }
+
+    static public List<OpenSeeWebcam> ListCameraDetails(bool includeBlackMagic) {
+        string jsonData = null;
+        string bmJsonData = null;
+        if (Environment.Is64BitProcess) {
+            jsonData = ListCameraDetails_x64();
+            if (includeBlackMagic)
+                bmJsonData = ListBlackMagicDetails_x64();
+        } else {
+            jsonData = ListCameraDetails_x86();
+            if (includeBlackMagic)
+                bmJsonData = ListBlackMagicDetails_x86();
+        }
+        if (dumpJsonStatic) {
+            UnityEngine.Debug.Log("Camera JSON: " + jsonData);
+            if (includeBlackMagic)
+                UnityEngine.Debug.Log("Blackmagic JSON: " + bmJsonData);
+        }
+        List<OpenSeeWebcam> details = JsonUtility.FromJson<OpenSeeWebcamList>("{\"list\":" + jsonData + "}").list;
+        foreach (var cam in details)
+            cam.type = OpenSeeWebcamType.DirectShow;
+        if (includeBlackMagic) {
+            List<OpenSeeWebcam> bmDetails = JsonUtility.FromJson<OpenSeeWebcamList>("{\"list\":" + bmJsonData + "}").list;
+            foreach (var cam in bmDetails)
+                cam.type = OpenSeeWebcamType.Blackmagic;
+            details.AddRange(bmDetails);
+        }
+        return details;
+    }
+    
+    void Start() {
+        Initialize(false);
+    }
+    
+    public void Initialize(bool includeBlackMagic) {
+        dumpJsonStatic = dumpJson;
+        cameras = ListCameraDetails(includeBlackMagic);
+        foreach (var camera in cameras)
+            camera.GetPrettyCapabilities();
+    }
+}
+}

Examples/OpenSeeWebcamInfo.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 2e9598f1218283b418c7e72604be0335
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: -50
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Licenses/DShowCapture.txt

@@ -0,0 +1,505 @@
+Modified by Emiliana_vt
+https://github.com/emilianavt/libdshowcapture
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 2.1, February 1999
+
+ Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+ 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+[This is the first released version of the Lesser GPL.  It also counts
+ as the successor of the GNU Library Public License, version 2, hence
+ the version number 2.1.]
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.
+
+  This license, the Lesser General Public License, applies to some
+specially designated software packages--typically libraries--of the
+Free Software Foundation and other authors who decide to use it.  You
+can use it too, but we suggest you first think carefully about whether
+this license or the ordinary General Public License is the better
+strategy to use in any particular case, based on the explanations below.
+
+  When we speak of free software, we are referring to freedom of use,
+not price.  Our General Public Licenses are designed to make sure that
+you have the freedom to distribute copies of free software (and charge
+for this service if you wish); that you receive source code or can get
+it if you want it; that you can change the software and use pieces of
+it in new free programs; and that you are informed that you can do
+these things.
+
+  To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights.  These restrictions translate to certain responsibilities for
+you if you distribute copies of the library or if you modify it.
+
+  For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you.  You must make sure that they, too, receive or can get the source
+code.  If you link other code with the library, you must provide
+complete object files to the recipients, so that they can relink them
+with the library after making changes to the library and recompiling
+it.  And you must show them these terms so they know their rights.
+
+  We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+  To protect each distributor, we want to make it very clear that
+there is no warranty for the free library.  Also, if the library is
+modified by someone else and passed on, the recipients should know
+that what they have is not the original version, so that the original
+author's reputation will not be affected by problems that might be
+introduced by others.
+
+  Finally, software patents pose a constant threat to the existence of
+any free program.  We wish to make sure that a company cannot
+effectively restrict the users of a free program by obtaining a
+restrictive license from a patent holder.  Therefore, we insist that
+any patent license obtained for a version of the library must be
+consistent with the full freedom of use specified in this license.
+
+  Most GNU software, including some libraries, is covered by the
+ordinary GNU General Public License.  This license, the GNU Lesser
+General Public License, applies to certain designated libraries, and
+is quite different from the ordinary General Public License.  We use
+this license for certain libraries in order to permit linking those
+libraries into non-free programs.
+
+  When a program is linked with a library, whether statically or using
+a shared library, the combination of the two is legally speaking a
+combined work, a derivative of the original library.  The ordinary
+General Public License therefore permits such linking only if the
+entire combination fits its criteria of freedom.  The Lesser General
+Public License permits more lax criteria for linking other code with
+the library.
+
+  We call this license the "Lesser" General Public License because it
+does Less to protect the user's freedom than the ordinary General
+Public License.  It also provides other free software developers Less
+of an advantage over competing non-free programs.  These disadvantages
+are the reason we use the ordinary General Public License for many
+libraries.  However, the Lesser license provides advantages in certain
+special circumstances.
+
+  For example, on rare occasions, there may be a special need to
+encourage the widest possible use of a certain library, so that it becomes
+a de-facto standard.  To achieve this, non-free programs must be
+allowed to use the library.  A more frequent case is that a free
+library does the same job as widely used non-free libraries.  In this
+case, there is little to gain by limiting the free library to free
+software only, so we use the Lesser General Public License.
+
+  In other cases, permission to use a particular library in non-free
+programs enables a greater number of people to use a large body of
+free software.  For example, permission to use the GNU C Library in
+non-free programs enables many more people to use the whole GNU
+operating system, as well as its variant, the GNU/Linux operating
+system.
+
+  Although the Lesser General Public License is Less protective of the
+users' freedom, it does ensure that the user of a program that is
+linked with the Library has the freedom and the wherewithal to run
+that program using a modified version of the Library.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.  Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library".  The
+former contains code derived from the library, whereas the latter must
+be combined with the library in order to run.
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any software library or other
+program which contains a notice placed by the copyright holder or
+other authorized party saying it may be distributed under the terms of
+this Lesser General Public License (also called "this License").
+Each licensee is addressed as "you".
+
+  A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+  The "Library", below, refers to any such software library or work
+which has been distributed under these terms.  A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language.  (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+  "Source code" for a work means the preferred form of the work for
+making modifications to it.  For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control compilation
+and installation of the library.
+
+  Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it).  Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+
+  1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+  You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+
+  2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) The modified work must itself be a software library.
+
+    b) You must cause the files modified to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    c) You must cause the whole of the work to be licensed at no
+    charge to all third parties under the terms of this License.
+
+    d) If a facility in the modified Library refers to a function or a
+    table of data to be supplied by an application program that uses
+    the facility, other than as an argument passed when the facility
+    is invoked, then you must make a good faith effort to ensure that,
+    in the event an application does not supply such function or
+    table, the facility still operates, and performs whatever part of
+    its purpose remains meaningful.
+
+    (For example, a function in a library to compute square roots has
+    a purpose that is entirely well-defined independent of the
+    application.  Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must
+    be optional: if the application does not supply it, the square
+    root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library.  To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License.  (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.)  Do not make any other change in
+these notices.
+
+  Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+  This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+  4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+  If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library".  Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+  However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library".  The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+  When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library.  The
+threshold for this to be true is not precisely defined by law.
+
+  If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work.  (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+  Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+
+  6. As an exception to the Sections above, you may also combine or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+  You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License.  You must supply a copy of this License.  If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License.  Also, you must do one
+of these things:
+
+    a) Accompany the work with the complete corresponding
+    machine-readable source code for the Library including whatever
+    changes were used in the work (which must be distributed under
+    Sections 1 and 2 above); and, if the work is an executable linked
+    with the Library, with the complete machine-readable "work that
+    uses the Library", as object code and/or source code, so that the
+    user can modify the Library and then relink to produce a modified
+    executable containing the modified Library.  (It is understood
+    that the user who changes the contents of definitions files in the
+    Library will not necessarily be able to recompile the application
+    to use the modified definitions.)
+
+    b) Use a suitable shared library mechanism for linking with the
+    Library.  A suitable mechanism is one that (1) uses at run time a
+    copy of the library already present on the user's computer system,
+    rather than copying library functions into the executable, and (2)
+    will operate properly with a modified version of the library, if
+    the user installs one, as long as the modified version is
+    interface-compatible with the version that the work was made with.
+
+    c) Accompany the work with a written offer, valid for at
+    least three years, to give the same user the materials
+    specified in Subsection 6a, above, for a charge no more
+    than the cost of performing this distribution.
+
+    d) If distribution of the work is made by offering access to copy
+    from a designated place, offer equivalent access to copy the above
+    specified materials from the same place.
+
+    e) Verify that the user has already received a copy of these
+    materials or that you have already sent this user a copy.
+
+  For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it.  However, as a special exception,
+the materials to be distributed need not include anything that is
+normally distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+  It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system.  Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+
+  7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+    a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities.  This must be distributed under the terms of the
+    Sections above.
+
+    b) Give prominent notice with the combined library of the fact
+    that part of it is a work based on the Library, and explaining
+    where to find the accompanying uncombined form of the same work.
+
+  8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License.  Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License.  However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+  9. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Library or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+  10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties with
+this License.
+
+  11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under any
+particular circumstance, the balance of the section is intended to apply,
+and the section as a whole is intended to apply in other circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License may add
+an explicit geographical distribution limitation excluding those countries,
+so that distribution is permitted only in or among countries not thus
+excluded.  In such case, this License incorporates the limitation as if
+written in the body of this License.
+
+  13. The Free Software Foundation may publish revised and/or new
+versions of the Lesser General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation.  If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+
+  14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission.  For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this.  Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+                            NO WARRANTY
+
+  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+           How to Apply These Terms to Your New Libraries
+
+  If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change.  You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms of the
+ordinary General Public License).
+
+  To apply these terms, attach the following notices to the library.  It is
+safest to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least the
+"copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the library's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the library, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the
+  library `Frob' (a library for tweaking knobs) written by James Random Hacker.
+
+  <signature of Ty Coon>, 1 April 1990
+  Ty Coon, President of Vice
+
+That's all there is to it!

Licenses/IL2cppStartProcess.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 June, Michał Petryka and Łukasz Jurczyk
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Licenses/Pytorch_Retinaface.txt

@@ -0,0 +1,21 @@
+MIT License 
+
+Copyright (c) 2019
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Licenses/Remedian.txt

@@ -0,0 +1,19 @@
+Copyright (c) 2016, Tim Menzies tim@menzies.us, MIT license v2.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Licenses/ThunderSVM.txt

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Licenses/haarcascade_frontalface_alt2.txt

@@ -1,41 +0,0 @@
-    Tree-based 20x20 gentle adaboost frontal face detector.
-    Created by Rainer Lienhart.
-
-////////////////////////////////////////////////////////////////////////////////////////
-
-  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
-
-  By downloading, copying, installing or using the software you agree to this license.
-  If you do not agree to this license, do not download, install,
-  copy or use the software.
-
-
-                        Intel License Agreement
-                For Open Source Computer Vision Library
-
- Copyright (C) 2000, Intel Corporation, all rights reserved.
- Third party copyrights are property of their respective owners.
-
- Redistribution and use in source and binary forms, with or without modification,
- are permitted provided that the following conditions are met:
-
-   * Redistribution's of source code must retain the above copyright notice,
-     this list of conditions and the following disclaimer.
-
-   * Redistribution's in binary form must reproduce the above copyright notice,
-     this list of conditions and the following disclaimer in the documentation
-     and/or other materials provided with the distribution.
-
-   * The name of Intel Corporation may not be used to endorse or promote products
-     derived from this software without specific prior written permission.
-
- This software is provided by the copyright holders and contributors "as is" and
- any express or implied warranties, including, but not limited to, the implied
- warranties of merchantability and fitness for a particular purpose are disclaimed.
- In no event shall the Intel Corporation or contributors be liable for any direct,
- indirect, incidental, special, exemplary, or consequential damages
- (including, but not limited to, procurement of substitute goods or services;
- loss of use, data, or profits; or business interruption) however caused
- and on any theory of liability, whether in contract, strict liability,
- or tort (including negligence or otherwise) arising in any way out of
- the use of this software, even if advised of the possibility of such damage.

Licenses/libminibmcapture.txt

@@ -0,0 +1,19 @@
+zlib License
+
+libminibmcapture (C) 2020 noorus
+
+This software is provided 'as-is', without any express or implied
+warranty.  In no event will the authors be held liable for any damages
+arising from the use of this software.
+
+Permission is granted to anyone to use this software for any purpose,
+including commercial applications, and to alter it and redistribute it
+freely, subject to the following restrictions:
+
+1. The origin of this software must not be misrepresented; you must not
+   claim that you wrote the original software. If you use this software
+   in a product, an acknowledgment in the product documentation would be
+   appreciated but is not required.
+2. Altered source versions must be plainly marked as such, and must not be
+   misrepresented as being the original software.
+3. This notice may not be removed or altered from any source distribution.

Licenses/libsvm.txt

@@ -0,0 +1,31 @@
+
+Copyright (c) 2000-2019 Chih-Chung Chang and Chih-Jen Lin
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither name of copyright holders nor the names of its contributors
+may be used to endorse or promote products derived from this software
+without specific prior written permission.
+
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Licenses/scikit-image.txt

@@ -0,0 +1,81 @@
+Copyright (C) 2019, the scikit-image team
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+ 3. Neither the name of skimage nor the names of its contributors may be
+    used to endorse or promote products derived from this software without
+    specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+skimage/_shared/version_requirements.py:_check_version
+
+    Copyright (c) 2013 The IPython Development Team
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice, this
+      list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+
+    * Neither the name of the copyright holder nor the names of its
+      contributors may be used to endorse or promote products derived from
+      this software without specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+    AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+    IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+    DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+    FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+    SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+    CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+    OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+skimage/_shared/version_requirements.py:is_installed:
+
+    Original Copyright (C) 2009-2011 Pierre Raybaut
+
+    Permission is hereby granted, free of charge, to any person obtaining
+    a copy of this software and associated documentation files (the
+    "Software"), to deal in the Software without restriction, including
+    without limitation the rights to use, copy, modify, merge, publish,
+    distribute, sublicense, and/or sell copies of the Software, and to
+    permit persons to whom the Software is furnished to do so, subject to
+    the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+    LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+    OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -1,22 +1,34 @@
+![OSF.png](https://raw.githubusercontent.com/emilianavt/OpenSeeFace/master/Images/OSF.png)
+
 # Overview
 
-This project implements a facial landmark detection model based on ShuffleNetV2.
+**Note**: This is a tracking library, **not** a stand-alone avatar puppeteering program. I'm also working on [VSeeFace](https://www.vseeface.icu/), which allows animating [VRM](https://vrm.dev/en/how_to_make_vrm/) and [VSFAvatar](https://www.youtube.com/watch?v=jhQ8DF87I5I) 3D models by using OpenSeeFace tracking. [VTube Studio](https://denchisoft.com/) uses OpenSeeFace for webcam based tracking to animate Live2D models. A renderer for the Godot engine can be found [here](https://github.com/virtual-puppet-project/vpuppr).
 
-As Pytorch 1.3 CPU inference speed on Windows is very low, the model was converted to ONNX format. Using [onnxruntime](https://github.com/microsoft/onnxruntime) it can run at 30 - 60 fps tracking a single face. A third and smaller model runs at over 60 fps tracking a single face on a single CPU core.
+This project implements a facial landmark detection model based on MobileNetV3.
+
+As Pytorch 1.3 CPU inference speed on Windows is very low, the model was converted to ONNX format. Using [onnxruntime](https://github.com/microsoft/onnxruntime) it can run at 30 - 60 fps tracking a single face. There are four models, with different speed to tracking quality trade-offs.
 
 If anyone is curious, the name is a silly pun on the open seas and seeing faces. There's no deeper meaning.
 
-Thanks to [@Virtual_Deat](https://twitter.com/Virtual_Deat) for helping me test everything.
+An up to date sample video can be found [here](https://www.youtube.com/watch?v=AaNap_ud_3I&vq=hd1080), showing the default tracking model's performance under different noise and light levels.
 
-[Unity sample video](https://twitter.com/emiliana_vt/status/1210622149314203648) | [Sample video](https://www.youtube.com/watch?v=AOPHiAp9DBE) | [Sample video](https://www.youtube.com/watch?v=-cBSuHGdBWQ)
+# Tracking quality
+
+Since the landmarks used by OpenSeeFace are a bit different from those used by other approaches (they are close to iBUG 68, with two less points in the mouth corners and quasi-3D face contours instead of face contours that follow the visible outline) it is hard to numerically compare its accuracy to that of other approaches found commonly in scientific literature. The tracking performance is also more optimized for making landmarks that are useful for animating an avatar than for exactly fitting the face image. For example, as long as the eye landmarks show whether the eyes are opened or closed, even if their location is somewhat off, they can still be useful for this purpose.
+
+From general observation, OpenSeeFace performs well in adverse conditions (low light, high noise, low resolution) and keeps tracking faces through a very wide range of head poses with relatively high stability of landmark positions. Compared to MediaPipe, OpenSeeFace landmarks remain more stable in challenging conditions and it accurately represents a wider range of mouth poses. However, tracking of the eye region can be less accurate.
+
+I ran OpenSeeFace on a sample clip from the video presentation for [3D Face Reconstruction with Dense Landmarks](https://microsoft.github.io/DenseLandmarks/) by Wood et al. to compare it to MediaPipe and their approach. You can watch the result [here](https://www.vseeface.icu/assets/media/OSFMediaPipe3DFR.mp4).
 
 # Usage
 
-The face tracking itself is done by the `facetracker.py` Python 3.7 script. It is a commandline program, so you should start it manually from cmd or write a batch file to start it.
+A sample Unity project for VRM based avatar animation can be found [here](https://github.com/emilianavt/OpenSeeFaceSample).
 
-The script will perform the tracking on a webcam or video file and send the tracking data over UDP. This design also allows tracking to be done on a separate PC from the one who uses the tracking information. This can be useful to enhance performance and to avoid accidentially revealing camera footage.
+The face tracking itself is done by the `facetracker.py` Python 3.7 script. It is a commandline program, so you should start it manually from cmd or write a batch file to start it. If you downloaded a release and are on Windows, you can run the `facetracker.exe` inside the `Binary` folder without having Python installed. You can also use the `run.bat` inside the `Binary` folder for a basic demonstration of the tracker.
 
-The provided `OpenSee` Unity component can receive these UDP packets and provides the received information through a public field called `trackingData`. The `OpenSeeShowPoints` component can visualize the landmark points of the first detected face. It also serves as an example. Please look at it to see how to properly make use of the `OpenSee` component. The UDP packets are received in a separate thread, so any components using the `trackingData` field of the `OpenSee` component should first copy the field and access this copy, because otherwise the information may get overwritten during processing.
+The script will perform the tracking on webcam input or video file and send the tracking data over UDP. This design also allows tracking to be done on a separate PC from the one who uses the tracking information. This can be useful to enhance performance and to avoid accidentially revealing camera footage.
+
+The provided `OpenSee` Unity component can receive these UDP packets and provides the received information through a public field called `trackingData`. The `OpenSeeShowPoints` component can visualize the landmark points of a detected face. It also serves as an example. Please look at it to see how to properly make use of the `OpenSee` component. Further examples are included in the `Examples` folder. The UDP packets are received in a separate thread, so any components using the `trackingData` field of the `OpenSee` component should first copy the field and access this copy, because otherwise the information may get overwritten during processing. This design also means that the field will keep updating, even if the `OpenSee` component is disabled.
 
 Run the python script with `--help` to learn about the possible options you can set.
 
@@ -24,7 +36,9 @@ Run the python script with `--help` to learn about the possible options you can
 
 A simple demonstration can be achieved by creating a new scene in Unity, adding an empty game object and both the `OpenSee` and `OpenSeeShowPoints` components to it. While the scene is playing, run the face tracker on a video file:
 
-    python facetracker.py --visualize 1 --max-threads 2 -c video.mp4
+    python facetracker.py --visualize 3 --pnp-points 1 --max-threads 4 -c video.mp4
+
+__Note__: If dependencies were installed using [poetry](https://python-poetry.org/), the commands have to be executed from a `poetry shell` or have to be prefixed with `poetry run`.
 
 This way the tracking script will output its own tracking visualization while also demonstrating the transmission of tracking data to Unity.
 
@@ -34,36 +48,89 @@ The included `OpenSeeLauncher` component allows starting the face tracker progra
 * `public bool StartTracker()` will start the tracker. If it is already running, it will shut down the running instance and start a new one with the current settings.
 * `public void StopTracker()` will stop the tracker. The tracker is stopped automatically when the application is terminated or the `OpenSeeLauncher` object is destroyed.
 
-The `OpenSeeLauncher` component uses WinAPI job objects to ensure that the tracker child process is terminated if the application crashes.
+The `OpenSeeLauncher` component uses WinAPI job objects to ensure that the tracker child process is terminated if the application crashes or closes without terminating the tracker process first.
 
 Additional custom commandline arguments should be added one by one into elements of `commandlineArguments` array. For example `-v 1` should be added as two elements, one element containing `-v` and one containing `1`, not a single one containing both parts.
 
+The included `OpenSeeIKTarget` component can be used in conjunction with FinalIK or other IK solutions to animate head motion.
+
+## Expression detection
+
+The `OpenSeeExpression` component can be added to the same component as the `OpenSeeFace` component to detect specific facial expressions. It has to be calibrated on a per-user basis. It can be controlled either through the checkboxes in the Unity Editor or through the equivalent public methods that can be found in its source code.
+
+To calibrate this system, you have to gather example data for each expression. If the capture process is going too fast, you can use the `recordingSkip` option to slow it down.
+
+The general process is as follows:
+
+* Type in a name for the expression you want to calibrate.
+* Make the expression and hold it, then tick the recording box.
+* Keep holding the expression and move your head around and turn it in various directions.
+* After a short while, start talking while doing so if the expression should be compatible with talking.
+* After doing this for a while, untick the recording box and work on capturing another expression.
+* Tick the train box and see if the expressions you gathered data for are detected accurately.
+* You should also get some statistics in the lower part of the component.
+* If there are issues with any expression being detected, keep adding data to it.
+
+To delete the captured data for an expression, type in its name and tick the "Clear" box.
+
+To save both the trained model and the captured training data, type in a filename including its full path in the "Filename" field and tick the "Save" box. To load it, enter the filename and tick the "Load" box.
+
+### Hints
+
+* A reasonable number of expressions is six, including the neutral one.
+* Before starting to capture expressions, make some faces and wiggle your eyebrows around, to warm up the feature detection part of the tracker.
+* Once you have a detection model that works decently, when using it take a moment to check all the expressions work as intended and add a little data if not.
+
 # General notes
 
 * The tracking seems to be quite robust even with partial occlusion of the face, glasses or bad lighting conditions.
-* There is no gaze direction tracking, and the model seems prone to thinking eyes are opened, even when they are closed.
+* The highest quality model is selected with `--model 3`, the fastest model with the lowest tracking quality is `--model 0`.
-* Depending on the frame rate, face tracking can easily use up a whole CPU core. At 30fps for a single face, it should still use less than 100% of one core on a decent CPU.
+* Lower tracking quality mainly means more rigid tracking, making it harder to detect blinking and eyebrow motion.
-* When setting the number of faces to track to a higher number than the number of faces actually in view, the OpenCV face detecter will attempt to find new faces every `--scan-every` frames. It can be quite slow, so try to set `--faces` no higher than the actual number of faces you are tracking.
+* Depending on the frame rate, face tracking can easily use up a whole CPU core. At 30fps for a single face, it should still use less than 100% of one core on a decent CPU. If tracking uses too much CPU, try lowering the frame rate. A frame rate of 20 is probably fine and anything above 30 should rarely be necessary.
+* When setting the number of faces to track to a higher number than the number of faces actually in view, the face detection model will run every `--scan-every` frames. This can slow things down, so try to set `--faces` no higher than the actual number of faces you are tracking.
 
 # Models
 
-Three pretrained models are included. Using the `--model` switch, it is possible to select them for tracking.
+Four pretrained face landmark models are included. Using the `--model` switch, it is possible to select them for tracking. The given fps values are for running the model on a single face video on a single CPU core. Lowering the frame rate would reduce CPU usage by a corresponding degree.
 
-* Model **0**: This is the original model I trained. It is the most reliable for general 3D pose tracking, but it has difficulties with eyebrow and eye positions.
+* Model **-1**: This model is for running on toasters, so it's a very very fast and very low accuracy model. (213fps without gaze tracking)
-* Model **1** (default): This model trades off a little bit of robustness for better facial feature tracking.
+* Model **0**: This is a very fast, low accuracy model. (68fps)
-* Model **2**: This is a smaller model that can run much faster, easily reaching 60 - 90 fps on a single CPU core. However, the tracking robustness and accuracy are lower.
+* Model **1**: This is a slightly slower model with better accuracy. (59fps)
+* Model **2**: This is a slower model with good accuracy. (50fps)
+* Model **3** (default): This is the slowest and highest accuracy model. (44fps)
+
+FPS measurements are from running on one core of my CPU.
+
+Pytorch weights for use with `model.py` can be found [here](https://mega.nz/file/vvYXlYQT#h7FpEg4tmOCJNxjpsDEw0JomJIkVGKwrt4OUV0RNDDU). Some unoptimized ONNX models can be found [here](https://github.com/emilianavt/OpenSeeFace/issues/48).
+
+# Results
+
+## Landmarks
+
+![Results1.png](https://raw.githubusercontent.com/emilianavt/OpenSeeFace/master/Images/Results1.png)
+
+![Results2.png](https://raw.githubusercontent.com/emilianavt/OpenSeeFace/master/Images/Results2.png)
+
+More samples: [Results3.png](https://raw.githubusercontent.com/emilianavt/OpenSeeFace/master/Images/Results3.png), [Results4.png](https://raw.githubusercontent.com/emilianavt/OpenSeeFace/master/Images/Results4.png)
+
+## Face detection
+
+The landmark model is quite robust with respect to the size and orientation of the faces, so the custom face detection model gets away with rougher bounding boxes than other approaches. It has a favorable speed to accuracy ratio for the purposes of this project.
+
+![EmiFace.png](https://raw.githubusercontent.com/emilianavt/OpenSeeFace/master/Images/EmiFace.png)
 
 # Release builds
 
 The builds in the release section of this repository contain a `facetracker.exe` inside a `Binary` folder that was built using `pyinstaller` and contains all required dependencies.
 
-To run it, at least the `models` and `escapi` folders have to be placed in the same folder as `facetracker.exe`.
+To run it, at least the `models` folder has to be placed in the same folder as `facetracker.exe`. Placing it in a common parent folder should work too.
 
 When distributing it, you should also distribute the `Licenses` folder along with it to make sure you conform to requirements set forth by some of the third party libraries. Unused models can be removed from redistributed packages without issue.
 
-# Dependencies
+The release builds contain a custom build of ONNX Runtime without telemetry.
+
+# Dependencies (Python 3.6 - 3.9)
 
-* Python 3.7
 * ONNX Runtime
 * OpenCV
 * Pillow
@@ -73,6 +140,22 @@ The required libraries can be installed using pip:
 
      pip install onnxruntime opencv-python pillow numpy
 
+Alternatively poetry can be used to 
+install all dependencies for this project in a separate virtual env:
+
+     poetry install
+
+# Dependencies
+
+* onnxruntime
+* OpenCV
+* Pillow
+* Numpy
+
+The required libraries can be installed using pip:
+ 
+	pip install onnxruntime opencv-python pillow numpy
+
 # References
 
 ## Training dataset
@@ -96,9 +179,18 @@ Additional training has been done on the WFLW dataset after reducing it to 66 po
       year = {2018}
     }
 
-The original model can still be selected with `--model 0` instead.
+For the training the gaze and blink detection model, the [MPIIGaze](https://www.mpi-inf.mpg.de/departments/computer-vision-and-machine-learning/research/gaze-based-human-computer-interaction/appearance-based-gaze-estimation-in-the-wild/) dataset was used. Additionally, around 125000 synthetic eyes generated with [UnityEyes](https://www.cl.cam.ac.uk/research/rainbow/projects/unityeyes/) were used during training.
 
-For the training the gaze and blink detection model, the WFLW dataset and [MRL Eye](http://mrl.cs.vsb.cz/eyedataset) datasets were used.
+It should be noted that additional custom data was also used during the training process and that the reference landmarks from the original datasets have been modified in certain ways to address various issues. It is likely not possible to reproduce these models with just the original LS3D-W and WFLW datasets, however the additional data is not redistributable.
+
+The heatmap regression based face detection model was trained on random 224x224 crops from the WIDER FACE dataset.
+
+	@inproceedings{yang2016wider,
+	  Author = {Yang, Shuo and Luo, Ping and Loy, Chen Change and Tang, Xiaoou},
+	  Booktitle = {IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
+	  Title = {WIDER FACE: A Face Detection Benchmark},
+	  Year = {2016}
+    }
 
 ## Algorithm
 
@@ -108,17 +200,42 @@ The algorithm is inspired by:
 * [Real-time Human Pose Estimation in the Browser with TensorFlow.js](https://blog.tensorflow.org/2018/05/real-time-human-pose-estimation-in.html)
 * [U-Net: Convolutional Networks for Biomedical Image Segmentation](https://lmb.informatik.uni-freiburg.de/people/ronneber/u-net/) by Olaf Ronneberger, Philipp Fischer, Thomas Brox
 * [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) by Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam
-* [ShuffleNet V2: Practical Guidelines for Efficient CNN Architecture Design](https://arxiv.org/abs/1807.11164) by Ningning Ma, Xiangyu Zhang, Hai-Tao Zheng, Jian Sun
+* [Searching for MobileNetV3](https://arxiv.org/abs/1905.02244) by Andrew Howard, Mark Sandler, Grace Chu, Liang-Chieh Chen, Bo Chen, Mingxing Tan, Weijun Wang, Yukun Zhu, Ruoming Pang, Vijay Vasudevan, Quoc V. Le, Hartwig Adam
 
-The ShuffleNet V2 code is taken from `torchvision`.
+The MobileNetV3 code was taken from [here](https://github.com/rwightman/gen-efficientnet-pytorch).
 
-For all training after the first model, a modified version of [Adaptive Wing Loss](https://github.com/tankrant/Adaptive-Wing-Loss) was used.
+For all training a modified version of [Adaptive Wing Loss](https://github.com/tankrant/Adaptive-Wing-Loss) was used.
 
 * [Adaptive Wing Loss for Robust Face Alignment via Heatmap Regression](https://arxiv.org/abs/1904.07399) by Xinyao Wang, Liefeng Bo, Li Fuxin
 
+For expression detection, [LIBSVM](https://www.csie.ntu.edu.tw/~cjlin/libsvm/) is used.
+
+Face detection is done using a custom heatmap regression based face detection model or RetinaFace.
+
+    @inproceedings{deng2019retinaface,
+      title={RetinaFace: Single-stage Dense Face Localisation in the Wild},
+      author={Deng, Jiankang and Guo, Jia and Yuxiang, Zhou and Jinke Yu and Irene Kotsia and Zafeiriou, Stefanos},
+      booktitle={arxiv},
+      year={2019}
+    }
+
+RetinaFace detection is based on [this](https://github.com/biubug6/Pytorch_Retinaface) implementation. The pretrained model was modified to remove unnecessary landmark detection and converted to ONNX format for a resolution of 640x640.
+
+# Thanks!
+
+Many thanks to everyone who helped me test things!
+
+* [@Virtual_Deat](https://twitter.com/Virtual_Deat), who also inspired me to start working on this.
+* [@ENiwatori](https://twitter.com/eniwatori) and family.
+* [@ArgamaWitch](https://twitter.com/ArgamaWitch)
+* [@AngelVayuu](https://twitter.com/AngelVayuu)
+* [@DapperlyYours](https://twitter.com/DapperlyYours)
+* [@comdost_art](https://twitter.com/comdost_art)
+* [@Ponoki_Chan](https://twitter.com/Ponoki_Chan)
+
 # License
 
-The code and model are distributed under the BSD 2-clause license. 
+The code and models are distributed under the BSD 2-clause license. 
 
-`models/haarcascade_frontalface_alt2.xml` is distributed under its own license. You can find licenses of third party libraries used for binary builds in the `Licenses` folder.
+You can find licenses of third party libraries used for binary builds in the `Licenses` folder.
 

Source/SVMWrapper.cpp

@@ -0,0 +1,211 @@
+// SVMWrapper.cpp : Defines the exported functions for the DLL application.
+//
+
+#include "stdafx.h"
+
+#include <cmath>
+#include "svm.h"
+
+using namespace std;
+
+extern "C" void __cdecl print_log(const char *str) {
+    return;
+}
+
+struct SVMWrapper {
+    struct svm_model *model;
+    int cols;
+    int classes;
+    double *means;
+    double *sdevs;
+    struct svm_parameter param;
+    struct svm_problem *problem;
+};
+
+typedef struct svm_node svmnode;
+
+svmnode **make_svmnode_array(float features[], int rows, int cols, double *means, double *sdevs, int rescale) {
+    svmnode **x = new svmnode*[rows];
+    svmnode *nodes = new svmnode[rows * (cols + 1)];
+
+    double row_factor = 1.0 / (double)rows;
+    if (rescale) {
+        for (int c = 0; c < cols; c++) {
+            means[c] = 0;
+            sdevs[c] = 0;
+        }
+        for (int r = 0; r < rows; r++) {
+            for (int c = 0; c < cols; c++) {
+                means[c] += (double)features[r * cols + c];
+            }
+        }
+        for (int c = 0; c < cols; c++) {
+            means[c] *= row_factor;
+        }
+        for (int r = 0; r < rows; r++) {
+            for (int c = 0; c < cols; c++) {
+                double sq_diff = ((double)features[r * cols + c] - means[c]);
+                sq_diff *= sq_diff;
+                sdevs[c] += sq_diff;
+            }
+        }
+        for (int c = 0; c < cols; c++) {
+            sdevs[c] *= row_factor;
+            sdevs[c] = sqrt(sdevs[c]);
+            sdevs[c] = 1.0 / sdevs[c];
+        }
+    }
+
+    for (int r = 0; r < rows; r++) {
+        x[r] = &(nodes[r * (cols + 1)]);
+        int idx = 0;
+        for (int c = 0; c < cols; c++) {
+            x[r][c].index = c + 1;
+            x[r][c].value = ((double)features[r * cols + c] - means[c]) * sdevs[c];
+        }
+        x[r][cols].index = -1;
+    }
+    return x;
+}
+
+void make_problem(struct svm_problem *problem, float features[], float labels[], int rows, int cols, double *means, double *sdevs) {
+    problem->l = rows;
+    problem->y = new double[rows];
+    for (int i = 0; i < rows; i++)
+        problem->y[i] = (double)labels[i];
+    problem->x = make_svmnode_array(features, rows, cols, means, sdevs, 1);
+}
+
+void destroy_svmnode_array(svmnode **x) {
+    delete x[0];
+    delete x;
+}
+
+void destroy_problem(struct svm_problem *problem) {
+    delete problem->y;
+    destroy_svmnode_array(problem->x);
+}
+
+extern "C" {
+    __declspec(dllexport) void *__cdecl trainModel(float features[], float labels[], float weights[], int rows, int cols, int classes, int probability, float C) {
+        if (!(rows > 0 && cols > 0))
+            return NULL;
+
+        svm_set_print_string_function(print_log);
+
+        struct SVMWrapper *sw = new SVMWrapper();
+        sw->cols = cols;
+        sw->classes = classes;
+        sw->means = new double[cols];
+        sw->sdevs = new double[cols];
+
+        sw->param.svm_type = C_SVC;
+        sw->param.kernel_type = RBF;
+        sw->param.degree = 3;
+        sw->param.gamma = 1.0 / (double)cols;
+        sw->param.coef0 = 0;
+        sw->param.nu = 0.5;
+        sw->param.cache_size = 100;
+        sw->param.C = C;
+        sw->param.eps = 0.001;
+        sw->param.p = 0.1;
+        sw->param.shrinking = 1;
+        sw->param.probability = probability;
+        sw->param.nr_weight = 0;
+        sw->param.weight_label = NULL;
+        sw->param.weight = NULL;
+
+        if (weights != NULL) {
+            sw->param.nr_weight = classes;
+            sw->param.weight_label = new int[classes];
+            sw->param.weight = new double[classes];
+            for (int i = 0; i < classes; i++) {
+                sw->param.weight_label[i] = i;
+                sw->param.weight[i] = (double)weights[i];
+            }
+        }
+
+        sw->problem = new struct svm_problem();
+        make_problem(sw->problem, features, labels, rows, cols, sw->means, sw->sdevs);
+        sw->model = svm_train(sw->problem, &sw->param);
+
+        if (weights != NULL) {
+            delete sw->param.weight_label;
+            delete sw->param.weight;
+        }
+
+        return sw;
+    }
+
+    __declspec(dllexport) void __cdecl predict(void *ptr, float features[], float predictions[], double probabilities[], int rows) {
+        struct SVMWrapper *sw = (struct SVMWrapper *)ptr;
+        int cols = sw->cols;
+
+        svmnode **x = make_svmnode_array(features, rows, cols, sw->means, sw->sdevs, 0);
+        for (int r = 0; r < rows; r++)
+            predictions[r] = (float)svm_predict_probability(sw->model, x[r], &(probabilities[r * sw->classes]));
+        destroy_svmnode_array(x);
+
+        return;
+    }
+
+    __declspec(dllexport) void *__cdecl loadModel(char *filename, int cols, int classes, double means[], double sdevs[]) {
+        struct SVMWrapper *sw = new SVMWrapper();
+        sw->cols = cols;
+        sw->model = svm_load_model(filename);
+        sw->classes = classes;
+        sw->means = new double[cols];
+        sw->sdevs = new double[cols];
+        for (int c = 0; c < cols; c++) {
+            sw->means[c] = means[c];
+            sw->sdevs[c] = sdevs[c];
+        }
+        return sw;
+    }
+
+    __declspec(dllexport) void *__cdecl loadModelString(char *modelString, int cols, int classes, double means[], double sdevs[]) {
+        struct SVMWrapper *sw = new SVMWrapper();
+        sw->cols = cols;
+        sw->model = svm_load_model_string(modelString);
+        sw->classes = classes;
+        sw->means = new double[cols];
+        sw->sdevs = new double[cols];
+        for (int c = 0; c < cols; c++) {
+            sw->means[c] = means[c];
+            sw->sdevs[c] = sdevs[c];
+        }
+        sw->problem = NULL;
+        return sw;
+    }
+
+    __declspec(dllexport) void __cdecl getScales(void *ptr, double means[], double sdevs[]) {
+        struct SVMWrapper *sw = (struct SVMWrapper *)ptr;
+        int cols = sw->cols;
+        for (int c = 0; c < cols; c++) {
+            means[c] = sw->means[c];
+            sdevs[c] = sw->sdevs[c];
+        }
+    }
+
+    __declspec(dllexport) char *__cdecl saveModelString (void *ptr) {
+        struct SVMWrapper *sw = (struct SVMWrapper *)ptr;
+        return svm_save_model_string(sw->model);
+    }
+
+    __declspec(dllexport) int __cdecl saveModel(void *ptr, char *filename) {
+        struct SVMWrapper *sw = (struct SVMWrapper *)ptr;
+        return svm_save_model(filename, sw->model);
+    }
+
+    __declspec(dllexport) void __cdecl destroyModel(void *ptr) {
+        struct SVMWrapper *sw = (struct SVMWrapper *)ptr;
+        delete sw->means;
+        delete sw->sdevs;
+        svm_free_and_destroy_model(&sw->model);
+        if (sw->problem != NULL) {
+            destroy_problem(sw->problem);
+            delete sw->problem;
+        }
+        delete sw;
+    }
+}

Source/ThunderSVMWrapper.cpp

@@ -0,0 +1,222 @@
+// ThunderSVMWrapper.cpp : Defines the exported functions for the DLL.
+//
+
+#include "pch.h"
+#include "framework.h"
+#include "ThunderSVMWrapper.h"
+#include "thundersvm/thundersvm.h"
+#include "thundersvm/model/svmmodel.h"
+#include "thundersvm/model/svc.h"
+#include <cmath>
+
+using namespace std;
+
+extern "C" void __cdecl print_log(const char *str) {
+    return;
+}
+
+struct problem {
+    int r;
+    int c;
+    DataSet::node2d x;
+    vector<double> y;
+    DataSet dataset;
+};
+
+struct SVMWrapper {
+    SVC *model;
+    int cols;
+    int classes;
+    double *means;
+    double *sdevs;
+    SvmParam param;
+    struct problem problem;
+};
+
+typedef struct svm_node svmnode;
+
+void make_svmnode_array(DataSet::node2d &dataset, float features[], int rows, int cols, double *means, double *sdevs, int rescale) {
+    double row_factor = 1.0 / (double)rows;
+    if (rescale) {
+        for (int c = 0; c < cols; c++) {
+            means[c] = 0;
+            sdevs[c] = 0;
+        }
+        for (int r = 0; r < rows; r++) {
+            for (int c = 0; c < cols; c++) {
+                means[c] += (double)features[r * cols + c];
+            }
+        }
+        for (int c = 0; c < cols; c++) {
+            means[c] *= row_factor;
+        }
+        for (int r = 0; r < rows; r++) {
+            for (int c = 0; c < cols; c++) {
+                double sq_diff = ((double)features[r * cols + c] - means[c]);
+                sq_diff *= sq_diff;
+                sdevs[c] += sq_diff;
+            }
+        }
+        for (int c = 0; c < cols; c++) {
+            sdevs[c] *= row_factor;
+            sdevs[c] = sqrt(sdevs[c]);
+            sdevs[c] = 1.0 / sdevs[c];
+        }
+    }
+
+    for (int r = 0; r < rows; r++) {
+        dataset[r] = vector<DataSet::node>(cols);
+        int idx = 0;
+        for (int c = 0; c < cols; c++) {
+            dataset[r][c].index = c + 1;
+            dataset[r][c].value = (float)((features[r * cols + c] - means[c]) * sdevs[c]);
+        }
+    }
+}
+
+void make_problem(struct problem &problem, float features[], float labels[], int rows, int cols, double *means, double *sdevs) {
+    problem.r = rows;
+    problem.c = cols;
+    problem.y = vector<double>(rows);
+    for (int i = 0; i < rows; i++)
+        problem.y[i] = (double)labels[i];
+    problem.x = DataSet::node2d(rows);
+    make_svmnode_array(problem.x, features, rows, cols, means, sdevs, 1);
+    problem.dataset = DataSet(problem.x, cols, problem.y);
+}
+
+void destroy_problem(struct problem &problem) {
+    problem.y.clear();
+    problem.x.clear();
+}
+
+extern "C" {
+    __declspec(dllexport) void *__cdecl trainModel(float features[], float labels[], float weights[], int rows, int cols, int classes, int probability, float C) {
+        if (!(rows > 0 && cols > 0))
+            return NULL;
+
+        //svm_set_print_string_function(print_log);
+
+        struct SVMWrapper *sw = new SVMWrapper();
+        sw->cols = cols;
+        sw->classes = classes;
+        sw->means = new double[cols];
+        sw->sdevs = new double[cols];
+
+        sw->param.svm_type = SvmParam::C_SVC;
+        sw->param.kernel_type = SvmParam::RBF;
+        sw->param.degree = 3;
+        sw->param.gamma = 1.0 / (double)cols;
+        sw->param.coef0 = 0;
+        sw->param.nu = 0.5;
+        sw->param.C = C;
+        sw->param.epsilon = 0.001;
+        sw->param.p = 0.1;
+        sw->param.probability = probability;
+        sw->param.nr_weight = 0;
+        sw->param.weight_label = NULL;
+        sw->param.weight = NULL;
+
+        if (weights != NULL) {
+            sw->param.nr_weight = classes;
+            sw->param.weight_label = new int[classes];
+            sw->param.weight = new double[classes];
+            for (int i = 0; i < classes; i++) {
+                sw->param.weight_label[i] = i;
+                sw->param.weight[i] = (double)weights[i];
+            }
+        }
+
+        make_problem(sw->problem, features, labels, rows, cols, sw->means, sw->sdevs);
+        sw->model = new SVC();//svm_train(sw->problem, &sw->param);
+        sw->model->train(sw->problem.dataset, sw->param);
+
+        if (weights != NULL) {
+            delete sw->param.weight_label;
+            delete sw->param.weight;
+        }
+
+        return sw;
+    }
+
+    __declspec(dllexport) void __cdecl predict(void *ptr, float features[], float predictions[], double probabilities[], int rows) {
+        struct SVMWrapper *sw = (struct SVMWrapper *)ptr;
+        int cols = sw->cols;
+
+        DataSet::node2d instances = DataSet::node2d(rows);
+        make_svmnode_array(instances, features, rows, sw->cols, sw->means, sw->sdevs, 0);
+        vector<double> predicted = sw->model->predict(instances, -1);
+        instances.clear();
+        for (int r = 0; r < rows; r++)
+            predictions[r] = (float)predicted[r];
+        if (probabilities != NULL && sw->param.probability) {
+            vector<float> probs = sw->model->get_prob_predict();
+            for (int r = 0; r < rows; r++)
+                for (int c = 0; c < sw->classes; c++)
+                    probabilities[c + r * sw->classes] = probs[c + r * sw->classes];
+        }
+        predicted.clear();
+
+        return;
+    }
+
+    __declspec(dllexport) void *__cdecl loadModel(char *filename, int cols, int classes, double means[], double sdevs[]) {
+        struct SVMWrapper *sw = new SVMWrapper();
+        sw->cols = cols;
+        sw->model = new SVC();
+        sw->model->load_from_file(filename);
+        sw->classes = classes;
+        sw->means = new double[cols];
+        sw->sdevs = new double[cols];
+        for (int c = 0; c < cols; c++) {
+            sw->means[c] = means[c];
+            sw->sdevs[c] = sdevs[c];
+        }
+        return sw;
+    }
+
+    __declspec(dllexport) void *__cdecl loadModelString(char *modelString, int cols, int classes, double means[], double sdevs[]) {
+        struct SVMWrapper *sw = new SVMWrapper();
+        sw->cols = cols;
+        sw->model = new SVC();
+        sw->model->load_from_string(modelString);
+        sw->classes = classes;
+        sw->means = new double[cols];
+        sw->sdevs = new double[cols];
+        for (int c = 0; c < cols; c++) {
+            sw->means[c] = means[c];
+            sw->sdevs[c] = sdevs[c];
+        }
+        return sw;
+    }
+
+    __declspec(dllexport) void __cdecl getScales(void *ptr, double means[], double sdevs[]) {
+        struct SVMWrapper *sw = (struct SVMWrapper *)ptr;
+        int cols = sw->cols;
+        for (int c = 0; c < cols; c++) {
+            means[c] = sw->means[c];
+            sdevs[c] = sw->sdevs[c];
+        }
+    }
+
+    __declspec(dllexport) const char *__cdecl saveModelString(void *ptr) {
+        struct SVMWrapper *sw = (struct SVMWrapper *)ptr;
+        string model = sw->model->save_to_string();
+        return _strdup(model.c_str());
+    }
+
+    __declspec(dllexport) int __cdecl saveModel(void *ptr, char *filename) {
+        struct SVMWrapper *sw = (struct SVMWrapper *)ptr;
+        sw->model->save_to_file(string(filename));
+        return 1;
+    }
+
+    __declspec(dllexport) void __cdecl destroyModel(void *ptr) {
+        struct SVMWrapper *sw = (struct SVMWrapper *)ptr;
+        delete sw->means;
+        delete sw->sdevs;
+        delete sw->model;
+        destroy_problem(sw->problem);
+        delete sw;
+    }
+}

Source/svm.h

@@ -0,0 +1,106 @@
+#ifndef _LIBSVM_H
+#define _LIBSVM_H
+
+#define LIBSVM_VERSION 324
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern int libsvm_version;
+
+struct svm_node
+{
+	int index;
+	double value;
+};
+
+struct svm_problem
+{
+	int l;
+	double *y;
+	struct svm_node **x;
+};
+
+enum { C_SVC, NU_SVC, ONE_CLASS, EPSILON_SVR, NU_SVR };	/* svm_type */
+enum { LINEAR, POLY, RBF, SIGMOID, PRECOMPUTED }; /* kernel_type */
+
+struct svm_parameter
+{
+	int svm_type;
+	int kernel_type;
+	int degree;	/* for poly */
+	double gamma;	/* for poly/rbf/sigmoid */
+	double coef0;	/* for poly/sigmoid */
+
+	/* these are for training only */
+	double cache_size; /* in MB */
+	double eps;	/* stopping criteria */
+	double C;	/* for C_SVC, EPSILON_SVR and NU_SVR */
+	int nr_weight;		/* for C_SVC */
+	int *weight_label;	/* for C_SVC */
+	double* weight;		/* for C_SVC */
+	double nu;	/* for NU_SVC, ONE_CLASS, and NU_SVR */
+	double p;	/* for EPSILON_SVR */
+	int shrinking;	/* use the shrinking heuristics */
+	int probability; /* do probability estimates */
+};
+
+//
+// svm_model
+//
+struct svm_model
+{
+	struct svm_parameter param;	/* parameter */
+	int nr_class;		/* number of classes, = 2 in regression/one class svm */
+	int l;			/* total #SV */
+	struct svm_node **SV;		/* SVs (SV[l]) */
+	double **sv_coef;	/* coefficients for SVs in decision functions (sv_coef[k-1][l]) */
+	double *rho;		/* constants in decision functions (rho[k*(k-1)/2]) */
+	double *probA;		/* pariwise probability information */
+	double *probB;
+	int *sv_indices;        /* sv_indices[0,...,nSV-1] are values in [1,...,num_traning_data] to indicate SVs in the training set */
+
+	/* for classification only */
+
+	int *label;		/* label of each class (label[k]) */
+	int *nSV;		/* number of SVs for each class (nSV[k]) */
+				/* nSV[0] + nSV[1] + ... + nSV[k-1] = l */
+	/* XXX */
+	int free_sv;		/* 1 if svm_model is created by svm_load_model*/
+				/* 0 if svm_model is created by svm_train */
+};
+
+struct svm_model *svm_train(const struct svm_problem *prob, const struct svm_parameter *param);
+void svm_cross_validation(const struct svm_problem *prob, const struct svm_parameter *param, int nr_fold, double *target);
+
+int svm_save_model(const char *model_file_name, const struct svm_model *model);
+char *svm_save_model_string(const svm_model *model);
+struct svm_model *svm_load_model(const char *model_file_name);
+struct svm_model *svm_load_model_string(char *model_string);
+
+int svm_get_svm_type(const struct svm_model *model);
+int svm_get_nr_class(const struct svm_model *model);
+void svm_get_labels(const struct svm_model *model, int *label);
+void svm_get_sv_indices(const struct svm_model *model, int *sv_indices);
+int svm_get_nr_sv(const struct svm_model *model);
+double svm_get_svr_probability(const struct svm_model *model);
+
+double svm_predict_values(const struct svm_model *model, const struct svm_node *x, double* dec_values);
+double svm_predict(const struct svm_model *model, const struct svm_node *x);
+double svm_predict_probability(const struct svm_model *model, const struct svm_node *x, double* prob_estimates);
+
+void svm_free_model_content(struct svm_model *model_ptr);
+void svm_free_and_destroy_model(struct svm_model **model_ptr_ptr);
+void svm_destroy_param(struct svm_parameter *param);
+
+const char *svm_check_parameter(const struct svm_problem *prob, const struct svm_parameter *param);
+int svm_check_probability_model(const struct svm_model *model);
+
+void svm_set_print_string_function(void (*print_func)(const char *));
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _LIBSVM_H */

Unity/OpenSee.cs

@@ -15,23 +15,36 @@ public class OpenSee : MonoBehaviour {
     [Tooltip("This is the port the server will listen for tracking packets on")]
     public int listenPort = 11573;
 
-    private const int nPoints = 66;
+    private const int nPoints = 68;
-    private const int packetFrameSize = 8 + 2 * 4 + 1 + 4 + 3 * 4 + 3 * 4 + 4 * 4 + 4 * 68 + 4 * 2 * 68 + 4 * 3 * 70;
+    private const int packetFrameSize = 8 + 4 + 2 * 4 + 2 * 4 + 1 + 4 + 3 * 4 + 3 * 4 + 4 * 4 + 4 * 68 + 4 * 2 * 68 + 4 * 3 * 70 + 4 * 14;
 
     [Header("Tracking data")]
     [Tooltip("This is an informational property that tells you how many packets have been received")]
     public int receivedPackets = 0;
     [Tooltip("This contains the actual tracking data")]
     public OpenSeeData[] trackingData = null;
+    
+    public bool listening { get; private set; } = false;
+    
+    [HideInInspector]
+    public float maxFit3DError = 100f;
 
     [System.Serializable]
     public class OpenSeeData {
         [Tooltip("The time this tracking data was captured at.")]
         public double time;
+        [Tooltip("This is the id of the tracked face. When tracking multiple faces, they might get reordered due to faces coming and going, but as long as tracking is not lost on a face, its id should stay the same. Face ids depend only on the order of first detection and locations of the faces.")]
+        public int id;
+        [Tooltip("This field gives the resolution of the camera or video being tracked.")]
+        public Vector2 cameraResolution;
         [Tooltip("This field tells you how likely it is that the right eye is open.")]
         public float rightEyeOpen;
         [Tooltip("This field tells you how likely it is that the left eye is open.")]
         public float leftEyeOpen;
+        [Tooltip("This field contains the rotation of the right eyeball.")]
+        public Quaternion rightGaze;
+        [Tooltip("This field contains the rotation of the left eyeball.")]
+        public Quaternion leftGaze;
         [Tooltip("This field tells you if 3D points have been successfully estimated from the 2D points. If this is false, do not rely on pose or 3D data.")]
         public bool got3DPoints;
         [Tooltip("This field contains the error for fitting the original 3D points. It shouldn't matter much, but it it is very high, something is probably wrong")]
@@ -50,11 +63,50 @@ public class OpenSee : MonoBehaviour {
         public Vector2[] points;
         [Tooltip("These are 3D points estimated from the 2D points. The should be rotation and translation compensated. There are 70 points with guesses for the eyeball center positions being added at the end of the 68 2D points.")]
         public Vector3[] points3D;
+        [Tooltip("This field contains a number of action unit like features.")]
+        public OpenSeeFeatures features;
+        
+        [System.Serializable]
+        public class OpenSeeFeatures {
+            [Tooltip("This field indicates whether the left eye is opened(0) or closed (-1). A value of 1 means open wider than normal.")]
+            public float EyeLeft;
+            [Tooltip("This field indicates whether the right eye is opened(0) or closed (-1). A value of 1 means open wider than normal.")]
+            public float EyeRight;
+            [Tooltip("This field indicates how steep the left eyebrow is, compared to the median steepness.")]
+            public float EyebrowSteepnessLeft;
+            [Tooltip("This field indicates how far up or down the left eyebrow is, compared to its median position.")]
+            public float EyebrowUpDownLeft;
+            [Tooltip("This field indicates how quirked the left eyebrow is, compared to its median quirk.")]
+            public float EyebrowQuirkLeft;
+            [Tooltip("This field indicates how steep the right eyebrow is, compared to the average steepness.")]
+            public float EyebrowSteepnessRight;
+            [Tooltip("This field indicates how far up or down the right eyebrow is, compared to its median position.")]
+            public float EyebrowUpDownRight;
+            [Tooltip("This field indicates how quirked the right eyebrow is, compared to its median quirk.")]
+            public float EyebrowQuirkRight;
+            [Tooltip("This field indicates how far up or down the left mouth corner is, compared to its median position.")]
+            public float MouthCornerUpDownLeft;
+            [Tooltip("This field indicates how far in or out the left mouth corner is, compared to its median position.")]
+            public float MouthCornerInOutLeft;
+            [Tooltip("This field indicates how far up or down the right mouth corner is, compared to its median position.")]
+            public float MouthCornerUpDownRight;
+            [Tooltip("This field indicates how far in or out the right mouth corner is, compared to its median position.")]
+            public float MouthCornerInOutRight;
+            [Tooltip("This field indicates how open or closed the mouth is, compared to its median pose.")]
+            public float MouthOpen;
+            [Tooltip("This field indicates how wide the mouth is, compared to its median pose.")]
+            public float MouthWide;
+        }
 
         public OpenSeeData() {
             confidence = new float[nPoints];
             points = new Vector2[nPoints];
-            points3D = new Vector3[nPoints];
+            points3D = new Vector3[nPoints + 2];
+        }
+        
+        private Vector3 swapX(Vector3 v) {
+            v.x = -v.x;
+            return v;
         }
 
         private float readFloat(byte[] b, ref int o) {
@@ -81,11 +133,14 @@ public class OpenSee : MonoBehaviour {
             Vector2 v = new Vector2(readFloat(b, ref o), readFloat(b, ref o));
             return v;
         }
-
+        
         public void readFromPacket(byte[] b, int o) {
             time = System.BitConverter.ToDouble(b, o);
             o += 8;
-            
+            id = System.BitConverter.ToInt32(b, o);
+            o += 4;
+
+            cameraResolution = readVector2(b, ref o);
             rightEyeOpen = readFloat(b, ref o);
             leftEyeOpen = readFloat(b, ref o);
 
@@ -101,44 +156,64 @@ public class OpenSee : MonoBehaviour {
             rawEuler = readVector3(b, ref o);
 
             rotation = rawEuler;
-            rotation.z = (rotation.z + 90) % 360;
+            rotation.z = (rotation.z - 90) % 360;
             rotation.x = -(rotation.x + 180) % 360;
 
             float x = readFloat(b, ref o);
             float y = readFloat(b, ref o);
             float z = readFloat(b, ref o);
-            translation = new Vector3(y, -x, z);
+            translation = new Vector3(-y, x, -z);
 
             for (int i = 0; i < nPoints; i++) {
                 confidence[i] = readFloat(b, ref o);
             }
-            
-            // Gaze tracker results are still bad, ignore them
-            readFloat(b, ref o); readFloat(b, ref o);
 
             for (int i = 0; i < nPoints; i++) {
                 points[i] = readVector2(b, ref o);
             }
-            
-            // Gaze tracker results are still bad, ignore them
-            readVector2(b, ref o); readVector2(b, ref o);
 
-            for (int i = 0; i < nPoints; i++) {
+            for (int i = 0; i < nPoints + 2; i++) {
                 points3D[i] = readVector3(b, ref o);
             }
             
-            // Gaze tracker results are still bad, ignore them
+            rightGaze = Quaternion.LookRotation(swapX(points3D[66]) - swapX(points3D[68])) * Quaternion.AngleAxis(180, Vector3.right) * Quaternion.AngleAxis(180, Vector3.forward);
-            readVector3(b, ref o); readVector3(b, ref o); readVector3(b, ref o); readVector3(b, ref o);
+            leftGaze = Quaternion.LookRotation(swapX(points3D[67]) - swapX(points3D[69])) * Quaternion.AngleAxis(180, Vector3.right) * Quaternion.AngleAxis(180, Vector3.forward);
+            
+            features = new OpenSeeFeatures();
+            features.EyeLeft = readFloat(b, ref o);
+            features.EyeRight = readFloat(b, ref o);
+            features.EyebrowSteepnessLeft = readFloat(b, ref o);
+            features.EyebrowUpDownLeft = readFloat(b, ref o);
+            features.EyebrowQuirkLeft = readFloat(b, ref o);
+            features.EyebrowSteepnessRight = readFloat(b, ref o);
+            features.EyebrowUpDownRight = readFloat(b, ref o);
+            features.EyebrowQuirkRight = readFloat(b, ref o);
+            features.MouthCornerUpDownLeft = readFloat(b, ref o);
+            features.MouthCornerInOutLeft = readFloat(b, ref o);
+            features.MouthCornerUpDownRight = readFloat(b, ref o);
+            features.MouthCornerInOutRight = readFloat(b, ref o);
+            features.MouthOpen = readFloat(b, ref o);
+            features.MouthWide = readFloat(b, ref o);
         }
     }
 
+    private Dictionary<int, OpenSeeData> openSeeDataMap;
     private Socket socket;
     private byte[] buffer;
     private Thread receiveThread = null;
     private volatile bool stopReception = false;
 
+    public OpenSeeData GetOpenSeeData(int faceId) {
+        if (openSeeDataMap == null)
+            return null;
+        if (!openSeeDataMap.ContainsKey(faceId))
+            return null;
+        return openSeeDataMap[faceId];
+    }
+
     void performReception() {
         EndPoint senderRemote = new IPEndPoint(IPAddress.Any, 0);
+        listening = true;
         while (!stopReception) {
             try {
                 int receivedBytes = socket.ReceiveFrom(buffer, SocketFlags.None, ref senderRemote);
@@ -147,39 +222,57 @@ public class OpenSee : MonoBehaviour {
                 }
                 receivedPackets++;
                 int i = 0;
-                OpenSeeData[] newData = new OpenSeeData[receivedBytes / packetFrameSize];
                 for (int offset = 0; offset < receivedBytes; offset += packetFrameSize) {
-                    newData[i] = new OpenSeeData();
+                    OpenSeeData newData = new OpenSeeData();
-                    newData[i].readFromPacket(buffer, offset);
+                    newData.readFromPacket(buffer, offset);
+                    openSeeDataMap[newData.id] = newData;
                     i++;
                 }
-                trackingData = newData;
+                trackingData = new OpenSeeData[openSeeDataMap.Count];
+                openSeeDataMap.Values.CopyTo(trackingData, 0);
             } catch { }
         }
     }
 
     void Start () {
+        if (openSeeDataMap == null)
+            openSeeDataMap = new Dictionary<int, OpenSeeData>();
         buffer = new byte[65535];
 
-        socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
+        if (socket == null) {
-        IPAddress ip;
+            socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
-        IPAddress.TryParse(listenAddress, out ip);
+            IPAddress ip;
-        socket.Bind(new IPEndPoint(ip, listenPort));
+            IPAddress.TryParse(listenAddress, out ip);
-        socket.ReceiveTimeout = 15;
+            socket.Bind(new IPEndPoint(ip, listenPort));
+            socket.ReceiveTimeout = 15;
+        }
 
         receiveThread = new Thread(() => performReception());
         receiveThread.Start();
     }
 
     void Update () {
+        if (receiveThread != null && !receiveThread.IsAlive) {
+            Start();
+        }
 	}
-
+    
-    void OnApplicationQuit() {
+    void EndReceiver() {
         if (receiveThread != null) {
             stopReception = true;
             receiveThread.Join();
             stopReception = false;
         }
+        if (socket != null)
+            socket.Close();
+    }
+
+    void OnApplicationQuit() {
+        EndReceiver();
+    }
+    
+    void OnDestroy() {
+        EndReceiver();
     }
 }
 

Unity/OpenSee.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 7b5af009eeb86f9469d2be8970e5a0f4
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/OpenSeeExpression.cs

@@ -0,0 +1,725 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Runtime.Serialization;
+using System.Runtime.Serialization.Formatters.Binary;
+using UnityEngine;
+
+namespace OpenSee {
+
+public class OpenSeeExpression : MonoBehaviour
+{
+    [Header("Settings")]
+    [Tooltip("This is the source of face tracking data. If it is not set, any OpenSee component on the current object is used.")]
+    public OpenSee openSee;
+    [Tooltip("When enabled ThunderSVM is used instead of LibSVM, which is usually faster, but may cause issues on some systems.")]
+    public bool useThunderSVM = false;
+    [Header("Simple mode")]
+    [Tooltip("When enabled, some simple heuristics will be used to detect expressions, rather than an SVM model. Outputs one of: neutral, fun, angry, surprise")]
+    public bool simpleMode = false;
+    [Tooltip("This smoothing factor is applied to the features used for simple expression detection.")]
+    [Range(0f, 1f)]
+    public float simpleSmoothing = 0.6f;
+    [Tooltip("This smoothing factor is applied to the features used for simple expression detection.")]
+    [Range(0f, 5f)]
+    public float simpleSensitivity = 1f;
+    [Header("Calibration")]
+    [Tooltip("This specifies which expression calibration data should be collected for.")]
+    public string calibrationExpression = "neutral";
+    [Tooltip("This specifies the id of the face for which data should be collected and predictions should be made. Face ids depend only on the order of first detection and locations of the faces.")]
+    public int faceId = 0;
+    [Tooltip("This keeps low numbers of misdetections from changing the expression, but introduces a bit of lag to expression changes. For example, a value of 1 means that a new expression has to be detected twice in a row, before the predicted expression changes.")]
+    public int expressionStabilizer = 1;
+    [Tooltip("Setting this to a value above 1 will record only every recordingSkip-th frame when capture calibration data. This can be useful with cameras that have a high framerate leading to only a short time for capturing varied expression information. Usually at least 20-30 seconds are required to capture a good variety of angles and expression variation, so this should be set accordingly when using the recording flag. At 1, no frames are skipped.")]
+    public int recordingSkip = 2;
+    [Tooltip("If allowOverRecording is enabled and expression data is recorded beyond 100%, this is used instead of recordingSkip to avoid quickly replacing the existing data.")]
+    public int overRecordingSkip = 5;
+    [Tooltip("This is the filename used for loading and saving expression data using the load and save flags.")]
+    public string filename = "";
+    [Tooltip("When enabled, training will be done in such a way that the model can output probabilities. This is a lot slower!")]
+    public bool enableProbabilityTraining = false;
+    [Tooltip("This is the SVM training C value.")]
+    public float C = 2f;
+    [Tooltip("This map allows you to set weights for different classes.")]
+    public Dictionary<string, float> weightMap = null;
+    [Header("Toggles")]
+    [Tooltip("When enabled, calibration data will be collected from the given OpenSee component.")]
+    public bool recording = false;
+    [Tooltip("When enabled, you can keep recording past 100%. When doing so, random frames in the already collected data will be overwritten with new data. To prevent quickly replacing all the collected data, overRecordingSkip is used instead of recordingSkip, which should be set to a higher value.")]
+    public bool allowOverRecording = false;
+    [Tooltip("When enabled, the calibration data collected for the current expression is cleared and this flag is set back to false.")]
+    public bool clear = false;
+    [Tooltip("When enabled, a new prediction model is trained and this flag is set back to false. Only expressions for which percentRecorded is 100% will be included in training.")]
+    public bool train = false;
+    [Tooltip("When enabled and a model is loaded, the current expression will be predicted every frame.")]
+    public bool predict = false;
+    [Tooltip("When enabled, the expression data from the specified filename will be loaded and this flag is set back to false.")]
+    public bool load = false;
+    [Tooltip("When enabled, the expression data will be saved to the specified filename and this flag is set back to false.")]
+    public bool save = false;
+    [Header("Point selection")]
+    [Tooltip("These settings can be used to train an expression detection model on a subset of points.")]
+    public PointSelection pointSelection;
+    [Header("Information")]
+    [Tooltip("This is the number of expressions for which calibration data was collected. The maximum number is 10.")]
+    public int expressionNumber = 0;
+    [Tooltip("This is the percentage of necessary training data collected for the current expression.")]
+    public float percentRecorded = 0.0f;
+    [Tooltip("This shows whether a model is ready to predict expressions with.")]
+    public bool modelReady = false;
+    [Tooltip("This is the time the current expression was detected.")]
+    public float expressionTime = 0f;
+    [Tooltip("This is currently detected expression.")]
+    public string expression = "";
+    [Tooltip("These are the probabilities of all expressions. The labels in order can be retrieved through the GetClassLabels methods.")]
+    public double[] probabilities;
+    [Header("Training statistics")]
+    [Tooltip("This shows the accuracy result of the last training run.")]
+    public float accuracy = 0.0f;
+    [Tooltip("This is the confusion matrix for the last training run's validation set.")]
+    public int[,] confusionMatrix = null;
+    [Tooltip("This is a pretty printed string representation of the confusion matrix.")]
+    public string confusionMatrixString = "";
+    [Tooltip("This shows any accuracy warnings that resulted from the last training run.")]
+    public string[] warnings = null;
+    
+    private float lastMouthCorner = 0f;
+    private float lastEyebrows = 0f;
+    private bool hadFun = false;
+    private bool hadAngry = false;
+    private bool hadSurprised = false;
+    float AdjustThreshold(bool active) {
+        if (active)
+            return 0.8f;
+        else
+            return 1f;
+    }
+    void ThresholdDetection() {
+        lastMouthCorner = lastMouthCorner * simpleSmoothing + (openSeeData.features.MouthCornerUpDownLeft + openSeeData.features.MouthCornerUpDownRight) * 0.5f * (1f - simpleSmoothing);
+        lastEyebrows = lastEyebrows * simpleSmoothing + (openSeeData.features.EyebrowUpDownLeft + openSeeData.features.EyebrowUpDownRight) * 0.5f * (1f - simpleSmoothing);
+        if (lastMouthCorner * simpleSensitivity < -0.2f * AdjustThreshold(hadFun)) {
+            expression = "fun";
+            hadFun = true;
+            hadSurprised = false;
+            hadAngry = false;
+        } else if (lastEyebrows * simpleSensitivity > 0.2f * AdjustThreshold(hadSurprised)) {
+            expression = "surprise";
+            hadFun = false;
+            hadSurprised = true;
+            hadAngry = false;
+        } else if (lastEyebrows * simpleSensitivity < -0.25f * AdjustThreshold(hadAngry) && lastMouthCorner * simpleSensitivity > -0.3f * (2f - AdjustThreshold(hadAngry))) {
+            expression = "angry";
+            hadFun = false;
+            hadSurprised = false;
+            hadAngry = true;
+        } else {
+            expression = "neutral";
+            hadFun = false;
+            hadSurprised = false;
+            hadAngry = false;
+        }
+        expressionTime = Time.time;
+    }
+
+    [Serializable]
+    public class PointSelection {
+        [Tooltip("When enabled, the points of the face contour will be used to determine the expression.")]
+        public bool pointsFaceContour = true;
+        [Tooltip("When enabled, the points of the right brow will be used to determine the expression.")]
+        public bool pointsBrowRight = true;
+        [Tooltip("When enabled, the points of the left brow will be used to determine the expression.")]
+        public bool pointsBrowLeft = true;
+        [Tooltip("When enabled, the points of the right eye will be used to determine the expression.")]
+        public bool pointsEyeRight = true;
+        [Tooltip("When enabled, the points of the left eye will be used to determine the expression.")]
+        public bool pointsEyeLeft = true;
+        [Tooltip("When enabled, the points of the nose will be used to determine the expression.")]
+        public bool pointsNose = true;
+        [Tooltip("When enabled, the points of the corners of the mouth will be used to determine the expression.")]
+        public bool pointsMouthCorner = true;
+        [Tooltip("When enabled, the points of the upper lip will be used to determine the expression.")]
+        public bool pointsLipUpper = true;
+        [Tooltip("When enabled, the points of the lower lip will be used to determine the expression.")]
+        public bool pointsLipLower = true;
+        [Tooltip("When enabled, certain manually designed features will be used to determine the expression.")]
+        public bool features = true;
+        [Tooltip("When enabled, the depth value of points is also used for training.")]
+        public bool includeDepth = false;
+    }
+    [Serializable]
+    private class OpenSeeExpressionRepresentation {
+        private Dictionary<string, List<float[]>> expressions;
+        private byte[] modelBytes = null;
+        private string[] classLabels = null;
+        private int[] indices = null;
+        private PointSelection pointSelection;
+        private bool thunderSVM = false;
+        //private bool newModel = false;
+
+        static public void LoadSerialized(byte[] modelBytes, out Dictionary<string, List<float[]>> expressions, out SVMModel model, out string[] classLabels, out int[] indices, ref PointSelection pointSelection) {
+            IFormatter formatter = new BinaryFormatter();
+            MemoryStream memoryStream = new MemoryStream(modelBytes);
+            memoryStream.Position = 0;
+            OpenSeeExpressionRepresentation oser;
+            using (GZipStream gzipStream = new GZipStream(memoryStream, CompressionMode.Decompress)) {
+                oser = formatter.Deserialize(gzipStream) as OpenSeeExpressionRepresentation;
+            }
+            expressions = oser.expressions;
+            /*if (oser.thunderSVM)
+                model = new ThunderSVMModel(oser.modelBytes);
+            else*/
+            model = new SVMModel(oser.modelBytes);
+            classLabels = oser.classLabels;
+            indices = oser.indices;
+            //pointSelection = oser.pointSelection;
+            if (indices == null) {
+                indices = new int[1 + 1 + 3 + 4 + 3 + 3 * 66];
+                for (int i = 0; i < 1 + 1 + 3 + 4 + 3 + 3 * 66; i++)
+                    indices[i] = i;
+            }
+            /*if (pointSelection == null)
+                pointSelection = new PointSelection();
+            if (!oser.newModel)
+                pointSelection.features = true;*/
+        }
+
+        static public byte[] ToSerialized(Dictionary<string, List<float[]>> expressions, SVMModel model, string[] classLabels, int[] indices, PointSelection pointSelection) {
+            OpenSeeExpressionRepresentation oser = new OpenSeeExpressionRepresentation();
+            oser.expressions = expressions;
+            oser.modelBytes = model.SaveModel();
+            oser.classLabels = classLabels;
+            oser.indices = indices;
+            oser.pointSelection = pointSelection;
+            oser.thunderSVM = model is ThunderSVMModel;
+            //oser.newModel = true;
+
+            IFormatter formatter = new BinaryFormatter();
+            MemoryStream memoryStream = new MemoryStream();
+            using (GZipStream gzipStream = new GZipStream(memoryStream, CompressionMode.Compress)) {
+                formatter.Serialize(gzipStream, oser);
+                gzipStream.Flush();
+            }
+            return memoryStream.ToArray();
+        }
+    }
+
+    private OpenSee.OpenSeeData openSeeData = null;
+    private Dictionary<string, List<float[]>> expressions;
+    private SVMModel model = null;
+    private string[] classLabels = null;
+    private int maxSamples = 1666;
+    // rightEyeOpen, leftEyeOpen, translation, rawQuaternion, rawEuler, confidence, points/(width, height), points3D, features
+    private int colsFull = 1 + 1 + 3 + 4 + 3 + /*66 + 2 * 66  +*/ 3 * 66 + 14;
+    private int colsBase = 1 + 1 + 3 + 4 + 3;
+    private int cols;
+    private double lastCapture = 0.0;
+    private float warningThreshold = 5;
+    private int currentPrediction = -1;
+    private int lastPrediction = -1;
+    private int lastPredictionCount = 0;
+    private int frameCount = 0;
+    private System.Random rnd;
+
+    private int[] indices;
+
+    private int[] indicesFaceContour = new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
+    private int[] indicesBrowRight = new int[] {17, 18, 19, 20, 21};
+    private int[] indicesBrowLeft = new int[] {22, 23, 24, 25, 26};
+    private int[] indicesEyeRight = new int[] {36, 37, 38, 39, 40, 41};
+    private int[] indicesEyeLeft = new int[] {42, 43, 44, 45, 46, 47};
+    private int[] indicesNose = new int[] {27, 28, 29, /*30, */31, 32, 33, 34, 35};
+    private int[] indicesMouthCorner = new int[] {58, 62};
+    private int[] indicesLipUpper = new int[] {48, 49, 50, 51, 52, 59, 60, 61};
+    private int[] indicesLipLower = new int[] {53, 54, 55, 56, 57, 63, 64, 65};
+
+    private void SelectPoints() {
+        List<int> indexList = new List<int>();
+        for (int i = 0; i < colsBase; i++)
+            if (i > 4) // Translation and (sometimes unavailable) eye open fields should not be used for training!
+                indexList.Add(i);
+        cols = indexList.Count;
+        if (pointSelection.pointsFaceContour)
+            foreach (int i in indicesFaceContour) {
+                indexList.Add(colsBase + i * 3);
+                indexList.Add(colsBase + i * 3 + 1);
+                cols += 2;
+                if (pointSelection.includeDepth) {
+                    indexList.Add(colsBase + i * 3 + 2);
+                    cols++;
+                }
+            }
+        if (pointSelection.pointsBrowRight)
+            foreach (int i in indicesBrowRight) {
+                indexList.Add(colsBase + i * 3);
+                indexList.Add(colsBase + i * 3 + 1);
+                cols += 2;
+                if (pointSelection.includeDepth) {
+                    indexList.Add(colsBase + i * 3 + 2);
+                    cols++;
+                }
+            }
+        if (pointSelection.pointsBrowLeft)
+            foreach (int i in indicesBrowLeft) {
+                indexList.Add(colsBase + i * 3);
+                indexList.Add(colsBase + i * 3 + 1);
+                cols += 2;
+                if (pointSelection.includeDepth) {
+                    indexList.Add(colsBase + i * 3 + 2);
+                    cols++;
+                }
+            }
+        if (pointSelection.pointsEyeRight)
+            foreach (int i in indicesEyeRight) {
+                indexList.Add(colsBase + i * 3);
+                indexList.Add(colsBase + i * 3 + 1);
+                cols += 2;
+                if (pointSelection.includeDepth) {
+                    indexList.Add(colsBase + i * 3 + 2);
+                    cols++;
+                }
+            }
+        if (pointSelection.pointsEyeLeft)
+            foreach (int i in indicesEyeLeft) {
+                indexList.Add(colsBase + i * 3);
+                indexList.Add(colsBase + i * 3 + 1);
+                cols += 2;
+                if (pointSelection.includeDepth) {
+                    indexList.Add(colsBase + i * 3 + 2);
+                    cols++;
+                }
+            }
+        if (pointSelection.pointsNose)
+            foreach (int i in indicesNose) {
+                indexList.Add(colsBase + i * 3);
+                indexList.Add(colsBase + i * 3 + 1);
+                cols += 2;
+                if (pointSelection.includeDepth) {
+                    indexList.Add(colsBase + i * 3 + 2);
+                    cols++;
+                }
+            }
+        if (pointSelection.pointsNose)
+            foreach (int i in indicesMouthCorner) {
+                indexList.Add(colsBase + i * 3);
+                indexList.Add(colsBase + i * 3 + 1);
+                cols += 2;
+                if (pointSelection.includeDepth) {
+                    indexList.Add(colsBase + i * 3 + 2);
+                    cols++;
+                }
+            }
+        if (pointSelection.pointsLipUpper)
+            foreach (int i in indicesLipUpper) {
+                indexList.Add(colsBase + i * 3);
+                indexList.Add(colsBase + i * 3 + 1);
+                cols += 2;
+                if (pointSelection.includeDepth) {
+                    indexList.Add(colsBase + i * 3 + 2);
+                    cols++;
+                }
+            }
+        if (pointSelection.pointsLipLower)
+            foreach (int i in indicesLipLower) {
+                indexList.Add(colsBase + i * 3);
+                indexList.Add(colsBase + i * 3 + 1);
+                cols += 2;
+                if (pointSelection.includeDepth) {
+                    indexList.Add(colsBase + i * 3 + 2);
+                    cols++;
+                }
+            }
+        if (pointSelection.features)
+            for (int i = 0; i < 14; i++) {
+                indexList.Add(colsBase + 66 * 3 + i);
+                cols++;
+            }
+        indices = indexList.ToArray();
+    }
+
+    private void ResetInfo() {
+        recording = false;
+        clear = false;
+        train = false;
+        modelReady = false;
+        accuracy = 0.0f;
+        confusionMatrix = null;
+        warnings = null;
+        percentRecorded = 0.0f;
+        expression = "";
+        confusionMatrixString = "";
+        lastPrediction = -1;
+        currentPrediction = -1;
+        lastPredictionCount = 0;
+        expressionTime = 0f;
+    }
+
+    void Start()
+    {
+        if (openSee == null)
+            openSee = GetComponent<OpenSee>();
+        if (openSee == null) {
+            ResetInfo();
+            return;
+        }
+        ResetInfo();
+        expressions = new Dictionary<string, List<float[]>>();
+        if (useThunderSVM)
+            model = new ThunderSVMModel();
+        else
+            model = new SVMModel();
+        rnd = new System.Random();
+    }
+
+    private float[] GetData(OpenSee.OpenSeeData t) {
+        float[] data = new float[colsFull];
+        data[0] = t.rightEyeOpen;
+        data[1] = t.leftEyeOpen;
+        data[2] = t.translation.x;
+        data[3] = t.translation.y;
+        data[4] = t.translation.z;
+        data[5] = t.rawQuaternion.x;
+        data[6] = t.rawQuaternion.y;
+        data[7] = t.rawQuaternion.z;
+        data[8] = t.rawQuaternion.w;
+        data[9] = t.rawEuler.x;
+        data[10] = t.rawEuler.y;
+        data[11] = t.rawEuler.z;
+        /*for (int i = 0; i < 66; i++)
+            data[12 + i] = t.confidence[i];
+        for (int i = 0; i < 66; i++) {
+            data[12 + 66 + 2 * i] = t.points[i].x / t.cameraResolution.x;
+            data[12 + 66 + 2 * i + 1] = t.points[i].y / t.cameraResolution.y;
+        }*/
+        for (int i = 0; i < 66; i++) {
+            data[12 + /*66 + 2 * 66 +*/ 3 * i] = t.points3D[i].x;
+            data[12 + /*66 + 2 * 66 +*/ 3 * i + 1] = t.points3D[i].y;
+            data[12 + /*66 + 2 * 66 +*/ 3 * i + 2] = t.points3D[i].z;
+        }
+        data[12 + 66 * 3 + 0] = t.features.EyeLeft;
+        data[12 + 66 * 3 + 1] = t.features.EyeRight;
+        data[12 + 66 * 3 + 2] = t.features.EyebrowSteepnessLeft;
+        data[12 + 66 * 3 + 3] = t.features.EyebrowUpDownLeft;
+        data[12 + 66 * 3 + 4] = t.features.EyebrowQuirkLeft;
+        data[12 + 66 * 3 + 5] = t.features.EyebrowSteepnessRight;
+        data[12 + 66 * 3 + 6] = t.features.EyebrowUpDownRight;
+        data[12 + 66 * 3 + 7] = t.features.EyebrowQuirkRight;
+        data[12 + 66 * 3 + 8] = t.features.MouthCornerUpDownLeft;
+        data[12 + 66 * 3 + 9] = t.features.MouthCornerInOutLeft;
+        data[12 + 66 * 3 + 10] = t.features.MouthCornerUpDownRight;
+        data[12 + 66 * 3 + 11] = t.features.MouthCornerInOutRight;
+        data[12 + 66 * 3 + 12] = t.features.MouthOpen;
+        data[12 + 66 * 3 + 13] = t.features.MouthWide;
+        return data;
+    }
+
+    public bool TrainModel() {
+        Debug.Log("---------------------");
+        train = false;
+        if (openSee == null) {
+            ResetInfo();
+            return false;
+        }
+        modelReady = false;
+        if (model is SVMModel && useThunderSVM)
+            model = new ThunderSVMModel();
+        else if (model is ThunderSVMModel && !useThunderSVM)
+            model = new SVMModel();
+        SelectPoints();
+        List<string> keys = new List<string>();
+        List<string> accuracyWarnings = new List<string>();
+        int samples = 0;
+        foreach (string key in expressions.Keys) {
+            List<float[]> list = new List<float[]>(expressions[key]);
+            list.RemoveAll(x => x.Length != colsFull);
+            if (list != null && list.Count == maxSamples) {
+                Debug.Log("[Training info] Adding expression " + key + " to training data.");
+                keys.Add(key);
+                samples += list.Count;
+            } else {
+                if (list != null && list.Count > 20) {
+                    Debug.Log("[Training warning] Expression " + key + " has little data and might be inaccurate. (" + list.Count + ")");
+                    accuracyWarnings.Add("Expression " + key + " has little data and might be inaccurate. (" + list.Count + ")");
+                    samples += list.Count;
+                    keys.Add(key);
+                } else {
+                    Debug.Log("[Training warning] Skipping expression " + key + " due to lack of collected data. (" + list.Count + ")");
+                    accuracyWarnings.Add("Skipping expression " + key + " due to lack of collected data. (" + list.Count + ")");
+                    continue;
+                }
+            }
+        }
+        int classes = keys.Count;
+        if (classes < 2 || classes > 10) {
+            Debug.Log("[Training error] The number of expressions that can be used for training is " + classes + ", which is either below 2 or higher than 10.");
+            accuracyWarnings.Add("[Training error] The number of expressions that can be used for training is " + classes + ", which is either below 2 or higher than 10.");
+            return false;
+        }
+        keys.Sort();
+        classLabels = keys.ToArray();
+        
+        float[] weights = null;
+        if (weightMap != null) {
+            weights = new float[classLabels.Length];
+            for (int i = 0; i < classLabels.Length; i++) {
+                if (weightMap.ContainsKey(classLabels[i]))
+                    weights[i] = weightMap[classLabels[i]];
+                else
+                    weights[i] = 1f;
+                Debug.Log("[Training info] Adding weight " + weights[i] + " for " + classLabels[i] + ".");
+            }
+        }
+        
+        Debug.Log("[Training info] Preparing trainig data.");
+        
+        int train_split = maxSamples * 3 / 4;
+        int test_split = maxSamples - train_split;
+        int rows_train = classes * train_split;
+        int rows_test = classes * test_split;
+        float[] X_train = new float[rows_train * cols];
+        float[] X_test = new float[rows_test * cols];
+        float[] y_train = new float[rows_train];
+        float[] y_test = new float[rows_test];
+        int i_train = 0;
+        int i_test = 0;
+        System.Random rnd = new System.Random();
+        for (int i = 0; i < classes; i++) {
+            List<float[]> list = new List<float[]>(expressions[keys[i]]);
+            list.RemoveAll(x => x.Length != colsFull);
+            int local_train_split = list.Count * 3 / 4;
+            test_split = list.Count - local_train_split;
+            for (int j = list.Count; j > 1;) {
+                j--;
+                int k = rnd.Next(j + 1);
+                float[] tmp = list[k];
+                list[k] = list[j];
+                list[j]= tmp;
+            }
+            int train_split_current = train_split;
+            if (classLabels[i] == "neutral")
+                train_split_current *= 1;
+            for (int j = 0; j < train_split_current; j++) {
+                float factor = 1f;
+                float adder = 0f;
+                /*if (j > train_split || j > list.Count) {
+                    factor = UnityEngine.Random.Range(0.98f, 1.02f);
+                    adder = UnityEngine.Random.Range(-0.02f, 0.02f);
+                }*/
+                for (int k = 0; k < cols; k++) {
+                    float v = list[j % local_train_split][indices[k]] * factor + adder;
+                    X_train[i_train * cols + k] = v;
+                }
+                y_train[i_train] = i;
+                i_train++;
+            }
+            for (int j = local_train_split; j < local_train_split + test_split; j++) {
+                for (int k = 0; k < cols; k++) {
+                    float v = list[j][indices[k]];
+                    X_test[i_test * cols + k] = v;
+                }
+                y_test[i_test] = i;
+                i_test++;
+            }
+        }
+        Debug.Log("[Training info] Preparation complete.");
+        int probability = 0;
+        if (enableProbabilityTraining)
+            probability = 1;
+        model.TrainModel(X_train, y_train, weights, i_train, cols, probability, C);
+        confusionMatrix = model.ConfusionMatrix(X_test, y_test, i_test, out accuracy);
+        confusionMatrixString = SVMModel.FormatMatrix(confusionMatrix, classLabels);
+        for (int label = 0; label < classes; label++) {
+            int total = 0;
+            for (int p = 0; p < classes; p++) {
+                total += confusionMatrix[label, p];
+            }
+            float error = 100f * (1f - ((float)confusionMatrix[label, label] / (float)total));
+            if (error > warningThreshold) {
+                accuracyWarnings.Add("[Training warning] The expression \"" + classLabels[label] + "\" is misclassified with a chance of " + error.ToString("0.00") + "%.");
+                Debug.Log("[Training warning] The expression \"" + classLabels[label] + "\" is misclassified with a chance of " + error.ToString("0.00") + "%.");
+            }
+        }
+        warnings = accuracyWarnings.ToArray();
+        modelReady = true;
+        Debug.Log("[Training info] Trained model.");
+        return true;
+    }
+
+    public bool PredictExpression() {
+        if (openSee == null)
+            return false;
+        if (!simpleMode && !(modelReady && model.Ready())) {
+            ResetInfo();
+            return false;
+        }
+        openSeeData = openSee.GetOpenSeeData(faceId);
+        if (openSeeData == null || openSeeData.time <= lastCapture)
+            return false;
+        if (simpleMode) {
+            ThresholdDetection();
+            return true;
+        }
+        float[] faceData = GetData(openSeeData);
+        float[] predictionData = new float[cols];
+        for (int i = 0; i < cols; i++)
+            predictionData[i] = faceData[indices[i]];
+        float[] prediction = model.Predict(predictionData, out probabilities, 1);
+        int predictedExpression = (int)Mathf.Round(prediction[0]);
+        if (predictedExpression == lastPrediction)
+            lastPredictionCount++;
+        else {
+            lastPrediction = predictedExpression;
+            lastPredictionCount = 1;
+        }
+        if (lastPredictionCount > expressionStabilizer) {
+            currentPrediction = lastPrediction;
+            expressionTime = Time.time;
+        }
+        if (currentPrediction >= 0 && currentPrediction <= classLabels.Length)
+            expression = classLabels[currentPrediction];
+        else
+            expression = "";
+        return true;
+    }
+
+    public bool CaptureCalibrationData() {
+        if (openSee == null) {
+            ResetInfo();
+            return false;
+        }
+        if (recordingSkip < 1)
+            recordingSkip = 1;
+        if (calibrationExpression == "")
+            return false;
+        if (expressions.ContainsKey(calibrationExpression) || expressions.Keys.Count < 10) {
+            openSeeData = openSee.GetOpenSeeData(faceId);
+            if (openSeeData == null || openSeeData.time <= lastCapture)
+                return false;
+            if (!expressions.ContainsKey(calibrationExpression)) {
+                expressions.Add(calibrationExpression, new List<float[]>());
+            }
+            List<float[]> list = expressions[calibrationExpression];
+            if (list.Count >= maxSamples) {
+                percentRecorded = 100f;
+                recording = allowOverRecording;
+                if (!recording)
+                    return false;
+            }
+            int skip = recordingSkip;
+            if (percentRecorded >= 100f)
+                skip = overRecordingSkip;
+            if (frameCount % skip != 0)
+                return true;
+            if (percentRecorded >= 100f) {
+                list[rnd.Next(maxSamples)] = GetData(openSeeData);
+            } else
+                list.Add(GetData(openSeeData));
+            percentRecorded = 100f * (float)list.Count/(float)maxSamples;
+            return true;
+        } else {
+            recording = false;
+            calibrationExpression = "";
+            return false;
+        }
+    }
+
+    public string[] GetClassLabels() {
+        List<string> labels = new List<string>();
+        foreach (string l in classLabels)
+            labels.Add(String.Copy(l));
+        return labels.ToArray();
+    }
+
+    public string[] GetExpressions() {
+        List<string> keys = new List<string>();
+        foreach (string key in expressions.Keys)
+            keys.Add(key);
+        keys.Sort();
+        return keys.ToArray();
+    }
+
+    public void ClearExpression() {
+            if (expressions.ContainsKey(calibrationExpression))
+                expressions.Remove(calibrationExpression);
+            clear = false;
+    }
+
+    public void LoadFromBytes(byte[] data) {
+        load = false;
+        if (openSee == null) {
+            ResetInfo();
+            return;
+        }
+        OpenSeeExpressionRepresentation.LoadSerialized(data, out expressions, out model, out classLabels, out indices, ref pointSelection);
+        cols = indices.Length;
+        if (model.Ready())
+            modelReady = true;
+    }
+
+    public byte[] SaveToBytes() {
+        save = false;
+        if (openSee == null) {
+            ResetInfo();
+            return null;
+        }
+        return OpenSeeExpressionRepresentation.ToSerialized(expressions, model, classLabels, indices, pointSelection);
+    }
+
+    public void LoadFromFile() {
+        load = false;
+        if (openSee == null) {
+            ResetInfo();
+            return;
+        }
+        byte[] data = File.ReadAllBytes(filename);
+        LoadFromBytes(data);
+    }
+
+    public void SaveToFile() {
+        save = false;
+        if (openSee == null) {
+            ResetInfo();
+            return;
+        }
+        byte[] data = SaveToBytes();
+        File.WriteAllBytes(filename, data);
+    }
+
+    void Update()
+    {
+        frameCount++;
+        if (openSee == null || expressions == null) {
+            ResetInfo();
+            return;
+        }
+        if (clear)
+            ClearExpression();
+        if (recording && calibrationExpression != "") {
+            CaptureCalibrationData();
+        }
+        if (train)
+            TrainModel();
+        expressionNumber = expressions.Keys.Count;
+        if (load)
+            LoadFromFile();
+        if (save)
+            SaveToFile();
+        if (!predict)
+            expression = "";
+        else if (simpleMode || modelReady)
+            PredictExpression();
+        else
+            predict = false;
+
+        if (expressions.ContainsKey(calibrationExpression))
+            percentRecorded = 100f * (float)expressions[calibrationExpression].Count/(float)maxSamples;
+        else
+            percentRecorded = 0f;
+
+        if (openSeeData == null)
+            return;
+       lastCapture = openSeeData.time;
+   }
+}
+
+}

Unity/OpenSeeExpression.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 9066de4c7793c1e40b212724f72fc064
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/OpenSeeIKTarget.cs

@@ -0,0 +1,331 @@
+using UnityEngine;
+
+namespace OpenSee {
+
+[DefaultExecutionOrder(-1)]
+public class OpenSeeIKTarget : MonoBehaviour
+{
+    [Header("Settings")]
+    [Tooltip("This is the source of face tracking data.")]
+    public OpenSee openSee;
+    [Tooltip("When this is set to another game object, its kinematic interpolation component will get updated from values of this.")]
+    public OpenSeeKinematicInterpolation kinematicInterpolation;
+    [Tooltip("When set, the this object will follow the specified transform instead and automatically recalibrate based on it.")]
+    public Transform externalTarget = null;
+    [Tooltip("This can be used to skip the first few tracking frames.")]
+    public int skipFirst = 5;
+    [Tooltip("When set to true, the next rotation and translation values will be set as zero rotation and zero translation.")]
+    public bool calibrate = true;
+    [Tooltip("This sets which face id to look for in the OpenSee data.")]
+    public int faceId = 0;
+    [Tooltip("When enabled, the IK target will not move up and down.")]
+    public bool preventVerticalMotion = false;
+    [Tooltip("When this is enabled, rotations and movement are mirrored.")]
+    public bool mirrorMotion = false;
+    [Tooltip("Often, the translation vector's scale is too high. Setting this somewhere between 0.05 to 0.3 seems stabilize things, but it also reduces the range of motion.")]
+    public float translationScale = 0.3f;
+    [Tooltip("This smoothed out the detected motion. Values can be between 0 (no smoothing) and 1 (no motion). The appropriate value probably depends on the camera's frame rate.")]
+    [Range(0, 1)]
+    public float smooth = 0.2f;
+    [Tooltip("When enabled, tracking will lag by one video capture frame, but the captured motion information is interpolated between frames, making things look smoother and less jittery and jumpy.")]
+    public bool interpolate = true;
+    [Tooltip("When set to true, the transform will be updated in FixedUpdate, otherwise it will be updated in Update.")]
+    public bool fixedUpdate = false;
+    [Tooltip("When set to true, the IK target's position and location will slowly drift back to its neutral pose.")]
+    public bool driftBack = false;
+    [Tooltip("This sets the strength of the drifting effect. At 0 there is now drifting, while the IK target will be static at 1.")]
+    public float driftFactor = 0.005f;
+    [Tooltip("This sets the minimum angle difference between two poses after which the pose will be considered an outlier.")]
+    [Range(0, 360)]
+    public float outlierThresholdAngle = 35f;
+    [Tooltip("This sets the number of seconds for which angle outliers are ignored before being accepted. Setting this to zero will disable outlier skipping.")]
+    public float outlierSkipPeriod = 0.4f;
+    [Tooltip("This sets the minimum distance difference between two poses after which the pose will be considered an outlier.")]
+    public float outlierThresholdDistance = 2f;
+    [Tooltip("This sets the number of seconds for which distance outliers are ignored before being accepted. Once this period elapses, automatic recalibration is triggered. Setting this to zero will disable outlier skipping.")]
+    public float outlierSkipPeriodDistance = 0.4f;
+    [Tooltip("If set to true, persistent positional outliers will cause automatic recalibration.")]
+    public bool outlierRecalibrate = false;
+    [Tooltip("If set to true, recalibration caused by positional outliers will keep the current position and rotation of the head.")]
+    public bool outlierKeepOffset = false;
+    [Header("Information")]
+    [Tooltip("This is the number of received tracking frames.")]
+    public int trackingFrames = 0;
+    [Tooltip("This is the current rotation calibration value as euler angles.")]
+    public Vector3 rotationOffset = new Vector3(0f, 0f, -90f);
+    [Tooltip("This is the current translation calibration.")]
+    public Vector3 translationOffset = new Vector3(0f, 0f, 0f);
+    [Tooltip("This is the average number of interpolated frames per received tracking data.")]
+    public float averageInterpolations = 0f;
+    [Tooltip("This is the number of outlier skipped tracking data frames so far.")]
+    public int outlierSkips = 0;
+    [Tooltip("This is the number of distance outlier skipped tracking data frames so far.")]
+    public int outlierSkipsDistance = 0;
+    [Tooltip("This is the number of times a recalibration was triggered due to positional outliers.")]
+    public int outlierCalibrations = 0;
+
+    private double updated = 0.0f;
+    private Quaternion dR = Quaternion.Euler(0f, 0f, -90f);
+    private Vector3 dT = new Vector3(0f, 0f, 0f);
+
+    private int interpolationCount = 1;
+    private int interpolateState = 0;
+    private float avgInterps = 1f;
+    private Quaternion lastR;
+    private Vector3 lastT;
+    private Quaternion currentR;
+    private Vector3 currentT;
+    private bool lastMirror = false;
+
+    private bool gotAccepted = false;
+    private Quaternion lastAccepted;
+    private bool skipped = false;
+    private float skipStartTime = -1f;
+    
+    private bool gotAcceptedPosition = false;
+    private Vector3 lastAcceptedPosition;
+    private bool skippedPosition = false;
+    private float skipStartTimePosition = -1f;
+    
+    private Transform lastExternalTarget = null;
+    
+    public void Calibrate() {
+        calibrate = true;
+    }
+    
+    public static Vector3 PrettyEuler(Vector3 last, Vector3 current) {
+        // Ew, but how would you do componentwise filtering/interpolation with quaternions?
+        if (Mathf.Abs(current.x - last.x) > Mathf.Abs((current.x - 360f) - last.x))
+            current.x -= 360f;
+        else if (Mathf.Abs(current.x - last.x) > Mathf.Abs((current.x + 360f) - last.x))
+            current.x += 360f;
+        if (Mathf.Abs(current.y - last.y) > Mathf.Abs((current.y - 360f) - last.y))
+            current.y -= 360f;
+        else if (Mathf.Abs(current.y - last.y) > Mathf.Abs((current.y + 360f) - last.y))
+            current.x += 360f;
+        if (Mathf.Abs(current.z - last.z) > Mathf.Abs((current.z - 360f) - last.z))
+            current.z -= 360f;
+        else if (Mathf.Abs(current.z - last.z) > Mathf.Abs((current.z + 360f) - last.z))
+            current.z += 360f;
+        return current;
+    }
+    
+    Vector3 FilterPos(Vector3 a, Vector3 b) {
+        return Vector3.Lerp(a, b, 1f - smooth);
+    }
+    
+    Quaternion FilterRot(Quaternion a, Quaternion b) {
+        return Quaternion.Lerp(a, b, 1f - smooth);
+    }
+
+    void Interpolate() {
+        if (!interpolate || interpolateState < 2)
+            return;
+        float t = Mathf.Clamp((float)interpolationCount / avgInterps, 0f, 1.5f);
+        transform.localPosition = Vector3.SlerpUnclamped(lastT, currentT, t);
+        transform.localRotation = Quaternion.SlerpUnclamped(lastR, currentR, t);
+        interpolationCount++;
+        //transform.localPosition = new Vector3(0.3f * ((Time.time/3f) % 3f) - 0.5f, 0.3f * Mathf.Sin(Mathf.Rad2Deg * ((Time.time/5f) % 3f)), 0f);
+        //transform.localRotation = Quaternion.identity * Quaternion.AngleAxis(20f * Mathf.Sin(Mathf.Rad2Deg * ((Time.time/5f) % 3f)) - 10f, Vector3.up);
+    }
+
+    Quaternion MirrorQuaternion(Quaternion q) {
+        return new Quaternion(-q.x, q.y, q.z, -q.w);
+    }
+
+    Vector3 MirrorTranslation(Vector3 v) {
+        return new Vector3(-v.x, v.y, v.z);
+    }
+
+    void RunUpdate() {
+        var openSeeData = openSee.GetOpenSeeData(faceId);
+        Quaternion convertedQuaternion;
+        Vector3 t;
+        Vector3 convertedTranslation;
+        
+        if (externalTarget == null) {
+            if (openSeeData == null || openSeeData.fit3DError > openSee.maxFit3DError)
+                return;
+            if (openSeeData.time > updated) {
+                updated = openSeeData.time;
+                trackingFrames++;
+                if (trackingFrames < skipFirst)
+                    return;
+            } else {
+                Interpolate();
+                return;
+            }
+            
+            convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w);
+            t = openSeeData.translation;
+            t.x = -t.x;
+            t.z = -t.z;
+            convertedTranslation = new Vector3(t.x, t.y, t.z);
+        } else {
+            convertedQuaternion = externalTarget.localRotation;
+            convertedTranslation = externalTarget.localPosition;
+            t = convertedTranslation;
+        }
+        
+        // Check for angular outliers
+        if (gotAccepted) {
+            float angularDifference = Quaternion.Angle(lastAccepted, convertedQuaternion);
+            if (angularDifference >= outlierThresholdAngle) {
+                if (!skipped) {
+                    if (outlierSkipPeriod >= 0.0000001f) {
+                        skipStartTime = Time.time;
+                        skipped = true;
+                    }
+                } else {
+                    if (Time.time > skipStartTime + outlierSkipPeriod) {
+                        lastAccepted = convertedQuaternion;
+                        skipped = false;
+                    }
+                }
+            } else {
+                skipped = false;
+                lastAccepted = convertedQuaternion;
+            }
+        } else {
+            lastAccepted = convertedQuaternion;
+            skipped = false;
+        }
+        gotAccepted = true;
+        if (skipped) {
+            outlierSkips += 1;
+            return;
+        }
+        
+        // Check for positional outliers
+        bool keepOffset = false;
+        if (gotAcceptedPosition) {
+            float distance = Vector3.Distance(t, lastAcceptedPosition);
+            if (distance > outlierThresholdDistance) {
+                if (!skippedPosition) {
+                    if (outlierSkipPeriodDistance >= 0.0000001f) {
+                        skipStartTimePosition = Time.time;
+                        skippedPosition = true;
+                    }
+                } else {
+                    if (Time.time > skipStartTimePosition + outlierSkipPeriodDistance) {
+                        lastAcceptedPosition = t;
+                        skippedPosition = false;
+                        if (outlierRecalibrate) {
+                            calibrate = true;
+                            keepOffset = outlierKeepOffset;
+                            outlierCalibrations += 1;
+                        }
+                    }
+                }
+            } else {
+                skippedPosition = false;
+                lastAcceptedPosition = t;
+            }
+        } else {
+            skippedPosition = false;
+            lastAcceptedPosition = t;
+        }
+        gotAcceptedPosition = true;
+        if (skippedPosition) {
+            outlierSkipsDistance += 1;
+            return;
+        }
+        
+        calibrate = calibrate || (externalTarget != lastExternalTarget);
+        lastExternalTarget = externalTarget;
+
+        if (calibrate) {
+            dR = convertedQuaternion;
+            dR = Quaternion.Inverse(dR);
+            dT = t;
+            rotationOffset = new Vector3(dR.eulerAngles.x, dR.eulerAngles.y, dR.eulerAngles.z);
+            translationOffset = new Vector3(dT.x, dT.y, dT.z);
+            lastAcceptedPosition = translationOffset;
+            skippedPosition = false;
+        }
+
+        if (mirrorMotion != lastMirror || (mirrorMotion && calibrate)) {
+            dR = Quaternion.Inverse(MirrorQuaternion(Quaternion.Inverse(dR)));
+            dT = MirrorTranslation(dT);
+            lastMirror = mirrorMotion;
+        }
+        if (calibrate && keepOffset) {
+            dR = Quaternion.Inverse(Quaternion.Inverse(transform.localRotation) * Quaternion.Inverse(dR));
+            dT -= transform.localPosition;
+        }
+        calibrate = false;
+
+        if (mirrorMotion) {
+            convertedQuaternion = MirrorQuaternion(convertedQuaternion);
+            t = MirrorTranslation(t);
+        }
+        
+        if (interpolateState > 1)
+            avgInterps = Mathf.Lerp(avgInterps, (float)interpolationCount, 0.15f);
+        interpolationCount = 0;
+        averageInterpolations = avgInterps;
+        
+        lastT = transform.localPosition;
+        lastR = transform.localRotation;
+        if (interpolateState > 0) {
+            currentT = FilterPos(transform.localPosition, (t - dT) * translationScale);
+            currentR = FilterRot(transform.localRotation, convertedQuaternion * dR);
+        } else {
+            currentT = (t - dT) * translationScale;
+            currentR = convertedQuaternion * dR;
+        }
+        if (interpolateState < 2)
+            interpolateState++;
+        
+        if (interpolate) {
+            Interpolate();
+        } else {
+            transform.localPosition = FilterPos(transform.localPosition, (t - dT) * translationScale);
+            transform.localRotation = FilterRot(transform.localRotation, convertedQuaternion * dR);
+        }
+
+       if (preventVerticalMotion) {
+           currentT.y = 0f;
+            Vector3 pos = transform.localPosition;
+            pos.y = 0f;
+            transform.localPosition = pos;
+        }
+
+        if (kinematicInterpolation != null && kinematicInterpolation.gameObject != gameObject) {
+            if (interpolate)
+                kinematicInterpolation.UpdateKI(updated, currentT, currentR);
+            else
+                kinematicInterpolation.UpdateKI(updated, transform.localPosition, transform.localRotation);
+        }
+        
+        if (driftBack) {
+            if (mirrorMotion) {
+                dT = Vector3.Lerp(dT, MirrorTranslation(convertedTranslation), driftFactor);
+                dR = Quaternion.Lerp(dR, Quaternion.Inverse(convertedQuaternion), driftFactor);
+            } else {
+                dT = Vector3.Lerp(dT, convertedTranslation, driftFactor);
+                dR = Quaternion.Lerp(dR, Quaternion.Inverse(convertedQuaternion), driftFactor);
+            }
+            rotationOffset = new Vector3(dR.eulerAngles.x, dR.eulerAngles.y, dR.eulerAngles.z);
+            translationOffset = new Vector3(dT.x, dT.y, dT.z);
+       }
+    }
+    
+    public void SetPreventVerticalMotion(bool v) {
+        preventVerticalMotion = v;
+    }
+
+    void FixedUpdate()
+    {
+        if (fixedUpdate)
+            RunUpdate();
+    }
+
+    void Update()
+    {
+        if (!fixedUpdate)
+            RunUpdate();
+    }
+}
+
+}

Unity/OpenSeeIKTarget.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 16cac0ed1df366e4cbb82f6c6a93ef1a
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: -5
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/OpenSeeInterpolation.cs

@@ -0,0 +1,232 @@
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+
+namespace OpenSee {
+    public class InterpolatedMap<T, V, U> where V : InterpolatedValue<U>, new() {
+        private Dictionary<T, V> map = new Dictionary<T, V>();
+        private float clamp = 1f;
+        private float smoothing = 0f;
+        private bool gotDefault = false;
+        private U defaultValue;
+
+        public void Clear() {
+            map.Clear();
+        }
+        
+        public void SetSmoothing(float smoothing) {
+            this.smoothing = smoothing;
+            foreach (var entry in map) {
+                if (entry.Value != null)
+                    entry.Value.SetSmoothing(smoothing);
+            }
+        }
+
+        public void SetClamp(float clamp) {
+            this.clamp = clamp;
+            foreach (var entry in map) {
+                if (entry.Value != null)
+                    entry.Value.SetClamp(clamp);
+            }
+        }
+        
+        void Prepare(T key) {
+            if (!map.ContainsKey(key)) {
+                map.Add(key, new V());
+                map[key].SetSmoothing(smoothing);
+                map[key].SetClamp(clamp);
+            }
+        }
+        
+        public void SetValue(T key, U v) {
+            Prepare(key);
+            map[key].SetValue(v);
+        }
+        
+        public void Store(T key, U v) {
+            Prepare(key);
+            map[key].UpdateValue(v);
+        }
+
+        public void Store(T key, U v, double nowT) {
+            if (!map.ContainsKey(key)) {
+                map.Add(key, new V());
+                map[key].SetSmoothing(smoothing);
+                map[key].SetClamp(clamp);
+            }
+            map[key].UpdateValue(v, nowT);
+        }
+        
+        public U GetDefault() {
+            if (gotDefault)
+                return defaultValue;
+            V tmp = new V();
+            defaultValue = tmp.GetDefault();
+            return defaultValue;
+        }
+        
+        public bool Check(T key) {
+            return map.ContainsKey(key);
+        }
+        
+        public U Get(T key) {
+            if (!map.ContainsKey(key))
+                return GetDefault();
+            return map[key].Interpolate();
+        }
+    }
+
+    abstract public class InterpolatedValue<V> {
+        private float interpolateT = 0f;
+        private float updateT = 0f;
+        private double lastT = 0f;
+        private double currentT = 0f;
+        private float clamp = 1f;
+        private float smoothing = 0f;
+        private int gotData = 0;
+
+        private bool haveDefault = false;
+        private V defaultValue;
+
+        private bool valueSet = false;
+        private V dataValue;
+        private V dataValueLast;
+
+        abstract public V ValueLerpUnclamped(V a, V b, float t);
+        abstract public V GetBuiltinDefault();
+        
+        public void SetDefault(V defaultValue) {
+            this.defaultValue = defaultValue;
+            haveDefault = true;
+        }
+        
+        public void ResetDefault() {
+            haveDefault = false;
+        }
+        
+        public V GetDefault() {
+            if (haveDefault)
+                return defaultValue;
+            return GetBuiltinDefault();
+        }
+        
+        public void SetSmoothing(float smoothing) {
+            this.smoothing = smoothing;
+        }
+
+        public void SetClamp(float clamp) {
+            this.clamp = clamp;
+        }
+        
+        public void SetValue(V v) {
+            dataValueLast = v;
+            dataValue = v;
+            valueSet = true;
+        }
+        
+        public InterpolatedValue(float smoothing = 0f, float clamp = 1f) {
+            this.smoothing = smoothing;
+            this.clamp = clamp;
+        }
+        
+        public void UpdateValue(V v) {
+            UpdateValue(v, Time.realtimeSinceStartup);
+        }
+        
+        public void UpdateValue(V v, double nowT) {
+            if (nowT < 0f)
+                return;
+            lock (this) {
+                if (nowT < currentT || gotData == 0) {
+                    // Reinitialize
+                    gotData = 1;
+                    updateT = Time.time;
+                    lastT = 0f;
+                    currentT = nowT;
+                    dataValueLast = GetDefault();
+                    dataValue = ValueLerpUnclamped(v, dataValueLast, smoothing);
+                } else if (nowT == currentT) {
+                    // Change value
+                    dataValue = ValueLerpUnclamped(v, dataValueLast, smoothing);
+                } else {
+                    // Regular operation
+                    updateT = Time.time;
+                    lastT = currentT;
+                    currentT = nowT;
+                    if (gotData < 2)
+                        gotData++;
+                    dataValueLast = dataValue;
+                    dataValue = ValueLerpUnclamped(v, dataValueLast, smoothing);
+                }
+            }
+        }
+        
+        public V Interpolate() {
+            lock (this) {
+                interpolateT = Time.time;
+                if (gotData < 2) {
+                    if (valueSet)
+                        return dataValue;
+                    else
+                        return GetDefault();
+                }
+                float t = Mathf.Min((interpolateT - updateT) / (float)(currentT - lastT), clamp);
+                return ValueLerpUnclamped(dataValueLast, dataValue, t);
+            }
+        }
+    }
+    
+    public class InterpolatedColor : InterpolatedValue<Color> {
+        public override Color ValueLerpUnclamped(Color a, Color b, float t) {
+            return Color.LerpUnclamped(a, b, t);
+        }
+        public override Color GetBuiltinDefault() {
+            return Color.clear;
+        }
+    }
+    
+    public class InterpolatedQuaternion : InterpolatedValue<Quaternion> {
+        public override Quaternion ValueLerpUnclamped(Quaternion a, Quaternion b, float t) {
+            return Quaternion.SlerpUnclamped(a, b, t);
+        }
+        public override Quaternion GetBuiltinDefault() {
+            return Quaternion.identity;
+        }
+    }
+
+    public class InterpolatedVector2 : InterpolatedValue<Vector2> {
+        public override Vector2 ValueLerpUnclamped(Vector2 a, Vector2 b, float t) {
+            return Vector2.LerpUnclamped(a, b, t);
+        }
+        public override Vector2 GetBuiltinDefault() {
+            return Vector2.zero;
+        }
+    }
+
+    public class InterpolatedVector3 : InterpolatedValue<Vector3> {
+        public override Vector3 ValueLerpUnclamped(Vector3 a, Vector3 b, float t) {
+            return Vector3.LerpUnclamped(a, b, t);
+        }
+        public override Vector3 GetBuiltinDefault() {
+            return Vector3.zero;
+        }
+    }
+
+    public class InterpolatedVector4 : InterpolatedValue<Vector4> {
+        public override Vector4 ValueLerpUnclamped(Vector4 a, Vector4 b, float t) {
+            return Vector4.LerpUnclamped(a, b, t);
+        }
+        public override Vector4 GetBuiltinDefault() {
+            return Vector4.zero;
+        }
+    }
+
+    public class InterpolatedFloat : InterpolatedValue<float> {
+        public override float ValueLerpUnclamped(float a, float b, float t) {
+            return Mathf.LerpUnclamped(a, b, t);
+        }
+        public override float GetBuiltinDefault() {
+            return 0f;
+        }
+    }
+}

Unity/OpenSeeInterpolation.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: d61e11f1b6900a7419383be1377f191b
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/OpenSeeJobManagement.cs

@@ -85,7 +85,6 @@ namespace OpenSee
         {
             return AddProcess(Process.GetProcessById(processId).Handle);
         }
-
     }
 
     #region Helper classes

Unity/OpenSeeJobManagement.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 3251308a085a78545a82e7f34ebc2d34
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/OpenSeeKinematicInterpolation.cs

@@ -0,0 +1,108 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+
+namespace OpenSee {
+
+public class OpenSeeKinematicInterpolation : MonoBehaviour
+{
+    public float interpolateClamp = 1f;
+    
+    private Vector3 lastPos;
+    private Vector3 lastPosV;
+    private double lastT;
+    
+    private Vector3 currentPos;
+    private Vector3 currentPosV;
+    private double currentT = 0f;
+    
+    private Vector3 lastRot = Vector3.zero;
+    private Vector3 lastRotV;
+    
+    private Vector3 currentRot;
+    private Vector3 currentRotV;
+    
+    private float dT;
+    private Vector3 coeffPosA;
+    private Vector3 coeffPosB;
+    private Vector3 coeffRotA;
+    private Vector3 coeffRotB;
+    
+    private double updateT;
+    private double interpolateT;
+    private int ready = 0;
+    
+    public void UpdateKI(double newT, Vector3 newPos, Quaternion newRot) {
+        Interpolate();
+        if (newT - currentT > 3f)
+            ready = 0;
+
+        currentPos = transform.localPosition;
+        currentRot = OpenSeeIKTarget.PrettyEuler(currentRot, transform.localRotation.eulerAngles);
+        float dTNow = (float)(interpolateT - updateT);
+        if (interpolateT - updateT > 0 || ready < 1) {
+            currentPosV = (currentPos - lastPos) / dT;
+            currentRotV = (currentRot - lastRot) / dT;
+        } else {
+            currentPosV = Vector3.zero;
+            currentRotV = Vector3.zero;
+        }
+
+        lastPos = currentPos;
+        lastPosV = currentPosV;
+        lastRot = currentRot;
+        lastRotV = currentRotV;
+        lastT = currentT;
+        
+        currentPos = newPos;
+        currentRot = OpenSeeIKTarget.PrettyEuler(lastRot, newRot.eulerAngles);
+        currentT = newT;
+        dT = (float)(currentT - lastT);
+        currentPosV = (currentPos - lastPos) / dT;
+        currentRotV = (currentRot - lastRot) / dT;
+
+        float dTPow2 = Mathf.Pow(dT, 2);
+        float dTPow3 = Mathf.Pow(dT, 3);
+        coeffPosA = (- (2f*(currentPosV * dT + 2f * dT * lastPosV - 3f * currentPos + 3f * lastPos)) / dTPow2) / 2f;
+        coeffPosB = ((6f*(currentPosV * dT + dT * lastPosV - 2f * currentPos + 2f * lastPos)) / dTPow3)/6f;
+        coeffRotA = (- (2f*(currentRotV * dT + 2f * dT * lastRotV - 3f * currentRot + 3f * lastRot)) / dTPow2) / 2f;
+        coeffRotB = ((6f*(currentRotV * dT + dT * lastRotV - 2f * currentRot + 2f * lastRot)) / dTPow3)/6f;
+        
+        updateT = Time.time;
+        if (ready <= 2)
+            ready++;
+    }
+    
+    float InterpolateValue(float dTNow, float dTNowPow2, float dTNowPow3, float a, float b, float x1, float x2, float v1, float v2) {
+        return x1 + v1 * dTNow + dTNowPow2 * a + dTNowPow3 * b;
+    }
+    
+    void Interpolate() {
+        interpolateT = Time.time;
+        if (interpolateT == updateT || ready <= 2)
+            return;
+        float dTNow = (float)(interpolateT - updateT);
+        if (dTNow >= dT) {
+            transform.localPosition = Vector3.LerpUnclamped(lastPos, currentPos, Mathf.Min((float)(dTNow / dT), interpolateClamp));
+            transform.localEulerAngles = Vector3.LerpUnclamped(lastRot, currentRot, Mathf.Min((float)(dTNow / dT), interpolateClamp));
+        } else {
+            float dTNowPow2 = Mathf.Pow(dTNow, 2);
+            float dTNowPow3 = Mathf.Pow(dTNow, 3);
+            float x = InterpolateValue(dTNow, dTNowPow2, dTNowPow3, coeffPosA.x, coeffPosB.x, lastPos.x, currentPos.x, lastPosV.x, currentPosV.x);
+            float y = InterpolateValue(dTNow, dTNowPow2, dTNowPow3, coeffPosA.y, coeffPosB.y, lastPos.y, currentPos.y, lastPosV.y, currentPosV.y);
+            float z = InterpolateValue(dTNow, dTNowPow2, dTNowPow3, coeffPosA.z, coeffPosB.z, lastPos.z, currentPos.z, lastPosV.z, currentPosV.z);
+            transform.localPosition = new Vector3(x, y, z);
+            x = InterpolateValue(dTNow, dTNowPow2, dTNowPow3, coeffRotA.x, coeffRotB.x, lastRot.x, currentRot.x, lastRotV.x, currentRotV.x);
+            y = InterpolateValue(dTNow, dTNowPow2, dTNowPow3, coeffRotA.y, coeffRotB.y, lastRot.y, currentRot.y, lastRotV.y, currentRotV.y);
+            z = InterpolateValue(dTNow, dTNowPow2, dTNowPow3, coeffRotA.z, coeffRotB.z, lastRot.z, currentRot.z, lastRotV.z, currentRotV.z);
+            transform.localEulerAngles = new Vector3(x, y, z);
+        }
+    }
+    
+    public void Update() {
+        Interpolate();
+    }
+}
+
+}

Unity/OpenSeeKinematicInterpolation.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: a4112b4be90fc9e4baee9eade65378c3
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: -1
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/OpenSeeLauncher.cs

@@ -2,6 +2,7 @@ using System;
 using System.Diagnostics;
 using System.IO;
 using System.Net;
+using System.Runtime.InteropServices;
 using System.Text;
 using System.Collections;
 using System.Collections.Generic;
@@ -11,6 +12,32 @@ using UnityEngine;
 namespace OpenSee {
 
 public class OpenSeeLauncher : MonoBehaviour {
+    #region DllImport
+    [DllImport("dshowcapture_x64", CallingConvention = CallingConvention.Cdecl, EntryPoint = "create_capture")]
+	private static extern System.IntPtr create_capture_x64();
+
+    [DllImport("dshowcapture_x64", CallingConvention = CallingConvention.Cdecl, EntryPoint = "get_devices")]
+	private static extern int get_devices_x64(System.IntPtr cap);
+
+	[DllImport("dshowcapture_x64", CallingConvention = CallingConvention.Cdecl, EntryPoint = "get_device")]
+	private static extern void get_device_x64(System.IntPtr cap, int deviceno, [Out] StringBuilder namebuffer, int bufferlength);
+
+    [DllImport("dshowcapture_x64", CallingConvention = CallingConvention.Cdecl, EntryPoint = "destroy_capture")]
+	private static extern void destroy_capture_x64(System.IntPtr cap);
+
+    [DllImport("dshowcapture_x86", CallingConvention = CallingConvention.Cdecl, EntryPoint = "create_capture")]
+	private static extern System.IntPtr create_capture_x86();
+
+    [DllImport("dshowcapture_x86", CallingConvention = CallingConvention.Cdecl, EntryPoint = "get_devices")]
+	private static extern int get_devices_x86(System.IntPtr cap);
+
+	[DllImport("dshowcapture_x86", CallingConvention = CallingConvention.Cdecl, EntryPoint = "get_device")]
+	private static extern void get_device_x86(System.IntPtr cap, int deviceno, [Out] StringBuilder namebuffer, int bufferlength);
+
+    [DllImport("dshowcapture_x86", CallingConvention = CallingConvention.Cdecl, EntryPoint = "destroy_capture")]
+	private static extern void destroy_capture_x86(System.IntPtr cap);
+    #endregion
+    
     [Header("Settings")]
     [Tooltip("If this is left empty, it will default to the OpenSee component on the same game object.")]
     public OpenSee openSeeTarget;
@@ -24,12 +51,26 @@ public class OpenSeeLauncher : MonoBehaviour {
     public bool autoListCameras = false;
     [Tooltip("If this is enabled, no output from the tracker will be printed to the debug console. This cannot be changed while the tracker is running.")]
     public bool dontPrint = true;
+    [Tooltip("If enabled, the next available port within 500 starting after the one given on the OpenSee component will be used.")]
+    public bool dynamicPort = false;
     [Tooltip("The path to \"facetracker.exe\".")]
     public string exePath = "facetracker.exe";
     [Tooltip("The path, where the .onnx model files can be found")]
     public string modelPath = "models\\";
     [Tooltip("Additional options that should be passed to the face tracker.")]
-    public string[] commandlineOptions = new string[] { "--silent", "1", "--max-threads", "2", "--model", "1" };
+    public List<string> commandlineOptions = new List<string>(new string[] { "--silent", "1", "--max-threads", "1" });
+    [Tooltip("This string will be appended at the end of the tracker options without quoting.")]
+    public string extraOptions = "";
+    [Tooltip("If enabled, the tracker commandline option string will be logged with Debug.Log.")]
+    public bool logCommandline = false;
+    [Tooltip("IL2CPP doesn't support Proocess.Start. When this is enabled, OpenSeeLauncher will create the tracking process by calling the necessary API functions directly through the DLLs. In this case, reading the standard output from the process will not be supported and it will instead be send to the logfiles set in pinvokeStdOut and pinvokeStdErr in the persistent data directory. It will also retrieve the camera list directly through the libdshowcapture DLLs, so make sure it is part of your Unity project.")]
+    public bool usePinvoke = false;
+    [Tooltip("When this is enabled, even if usePinvoke is disabled, the camera list will be retrieved through libdshowcapture DLLs directly, which can be faster. Make sure the DLLs are in your Unity project.")]
+    public bool usePinvokeListCameras = false;
+    [Tooltip("This is the standard output log file's name in the persistent data directory.")]
+    public string pinvokeStdOut = "output.txt";
+    [Tooltip("This is the standard error log file's name in the persistent data directory.")]
+    public string pinvokeStdErr = "error.txt";
     [Header("Runtime information")]
     [Tooltip("This field shows if the tracker is currently alive.")]
     public bool trackerAlive = false;
@@ -38,7 +79,34 @@ public class OpenSeeLauncher : MonoBehaviour {
     private Process trackerProcess = null;
     private StringBuilder trackerSB = null;
     private bool dontPrintNow = false;
+    private System.IntPtr processHandle = System.IntPtr.Zero;
+    private System.IntPtr processStdOut = System.IntPtr.Zero;
+    private System.IntPtr processStdErr = System.IntPtr.Zero;
     private Job job = null;
+    
+    public void UnsetOption(string name, bool hasArgument) {
+        int count = 1;
+        if (hasArgument)
+            count++;
+        int found = -1;
+        int i = 0;
+        foreach (string arg in commandlineOptions) {
+            if (arg == name) {
+                found = i;
+                break;
+            }
+            i++;
+        }
+        if (found > -1)
+            commandlineOptions.RemoveRange(found, count);
+    }
+    
+    public void SetOption(string name, string argument) {
+        UnsetOption(name, argument != null);
+        commandlineOptions.Add(name);
+        if (argument != null)
+            commandlineOptions.Add(argument);
+    }
 
     private bool CheckSetup(bool requireTarget) {
         if (openSeeTarget == null)
@@ -52,10 +120,10 @@ public class OpenSeeLauncher : MonoBehaviour {
             UnityEngine.Debug.LogError("Facetracker executable cannot be found.");
             return false;
         }
-        if (!Directory.Exists(modelPath)) {
+        /*if (!Directory.Exists(modelPath)) {
             UnityEngine.Debug.LogError("Model directory cannot be found.");
             return false;
-        }
+        }*/
         if (requireTarget) {
             ip = openSeeTarget.listenAddress;
             if (!IPAddress.TryParse(ip, out address)) {
@@ -85,10 +153,43 @@ public class OpenSeeLauncher : MonoBehaviour {
         return true;
     }
     
+    private string[] ListCameras_x64() {
+		List<string> cameras = new List<string>();
+        System.IntPtr cap = create_capture_x64();
+		int count = get_devices_x64(cap);
+		for (int i = 0; i < count; i++) {
+			StringBuilder namebuffer = new StringBuilder(2048);
+			get_device_x64(cap, i, namebuffer, 2048);
+			cameras.Add(namebuffer.ToString());
+		}
+        destroy_capture_x64(cap);
+		return cameras.ToArray();
+    }
+    
+    private string[] ListCameras_x86() {
+		List<string> cameras = new List<string>();
+        System.IntPtr cap = create_capture_x86();
+		int count = get_devices_x86(cap);
+		for (int i = 0; i < count; i++) {
+			StringBuilder namebuffer = new StringBuilder(2048);
+			get_device_x86(cap, i, namebuffer, 2048);
+			cameras.Add(namebuffer.ToString());
+		}
+        destroy_capture_x86(cap);
+		return cameras.ToArray();
+    }
+    
     public string[] ListCameras() {
+        if (usePinvoke || usePinvokeListCameras) {
+            if (Environment.Is64BitProcess)
+                return ListCameras_x64();
+            else
+                return ListCameras_x86();
+        }
+        
         if (!CheckSetup(false))
             return null;
-        
+
         StringBuilder stringBuilder;
         ProcessStartInfo processStartInfo;
         Process process;
@@ -122,6 +223,48 @@ public class OpenSeeLauncher : MonoBehaviour {
         return cameras;
     }
     
+    public float[] Benchmark(int threads) {
+        if (!CheckSetup(false))
+            return null;
+
+        StringBuilder stringBuilder;
+        ProcessStartInfo processStartInfo;
+        Process process;
+
+        stringBuilder = new StringBuilder();
+
+        processStartInfo = new ProcessStartInfo();
+        processStartInfo.CreateNoWindow = true;
+        processStartInfo.RedirectStandardOutput = true;
+        processStartInfo.RedirectStandardInput = true;
+        processStartInfo.RedirectStandardError = true;
+        processStartInfo.UseShellExecute = false;
+        processStartInfo.FileName = exePath;
+        processStartInfo.Arguments = "--benchmark 1 --priority 4 --max-threads " + threads.ToString();
+        
+        process = new Process();
+        process.StartInfo = processStartInfo;
+        process.EnableRaisingEvents = true;
+        process.OutputDataReceived += new DataReceivedEventHandler(delegate(object sender, DataReceivedEventArgs e) {
+            stringBuilder.Append(e.Data);
+            stringBuilder.Append("\n");
+        });
+        process.Start();
+        job.AddProcess(process.Handle);
+        process.BeginOutputReadLine();
+        process.WaitForExit();
+        process.CancelOutputRead();
+        
+        string[] lines = stringBuilder.ToString().Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.RemoveEmptyEntries);
+        float[] results = new float[lines.Length];
+        
+        for (int i = 0; i < lines.Length; i++) {
+            results[i] = Single.Parse(lines[i]);
+        }
+        
+        return results;
+    }
+    
     // From: http://csharptest.net/529/how-to-correctly-escape-command-line-arguments-in-c/index.html
     private static string EscapeArguments(params string[] args)
     {
@@ -151,23 +294,35 @@ public class OpenSeeLauncher : MonoBehaviour {
         if (!CheckSetup(true))
             return false;
 
-        ProcessStartInfo processStartInfo;
-        
-        processStartInfo = new ProcessStartInfo();
-        processStartInfo.CreateNoWindow = true;
-        processStartInfo.RedirectStandardOutput = true;
-        processStartInfo.RedirectStandardInput = true;
-        processStartInfo.RedirectStandardError = true;
-        processStartInfo.UseShellExecute = false;
-        processStartInfo.FileName = exePath;
-        
         List<string> arguments = new List<string>();
         arguments.Add("--ip");
         arguments.Add(ip);
+        
+        if (dynamicPort && !openSeeTarget.listening) {
+            System.Net.IPEndPoint[] inUse = System.Net.NetworkInformation.IPGlobalProperties.GetIPGlobalProperties().GetActiveUdpListeners();
+            Array.Sort(inUse, delegate(System.Net.IPEndPoint a, System.Net.IPEndPoint b) { return a.Port.CompareTo(b.Port); });
+            int port = openSeeTarget.listenPort + 1;
+            for (int i = 0; i < inUse.Length; i++) {
+                if (inUse[i].Port < port)
+                    continue;
+                if (inUse[i].Port == port) {
+                    if (port <= openSeeTarget.listenPort + 500 && port <= IPEndPoint.MaxPort) {
+                        port++;
+                        continue;
+                    } else {
+                        return false;
+                    }
+                }
+                break;
+            }
+            openSeeTarget.listenPort = port;
+        }
+
         arguments.Add("--port");
         arguments.Add(openSeeTarget.listenPort.ToString());
         arguments.Add("--model-dir");
         arguments.Add(modelPath);
+        
         arguments.Add("--capture");
         if (videoPath != "" && File.Exists(videoPath))
             arguments.Add(videoPath);
@@ -177,45 +332,76 @@ public class OpenSeeLauncher : MonoBehaviour {
             arguments.Add(argument);
         }
         string argumentString = EscapeArguments(arguments.ToArray());
-        processStartInfo.Arguments = argumentString;
+        if (extraOptions != "")
+            argumentString = argumentString + " " + extraOptions;
         
-        if (trackerProcess != null && !trackerProcess.HasExited) {
+        if (logCommandline)
-            trackerProcess.CloseMainWindow();
+            UnityEngine.Debug.Log("Starting tracker: " + argumentString);
-            trackerProcess.Close();
+
+        StopTracker();
+
+        if (!usePinvoke) {
+            ProcessStartInfo processStartInfo;
+            processStartInfo = new ProcessStartInfo();
+            processStartInfo.CreateNoWindow = true;
+            processStartInfo.RedirectStandardOutput = true;
+            processStartInfo.RedirectStandardInput = true;
+            processStartInfo.RedirectStandardError = true;
+            processStartInfo.UseShellExecute = false;
+            processStartInfo.FileName = exePath;
+            processStartInfo.Arguments = argumentString;
+            
+            trackerSB = new StringBuilder();
+            trackerProcess = new Process();
+            trackerProcess.StartInfo = processStartInfo;
+            dontPrintNow = dontPrint;
             if (!dontPrintNow) {
-                try {
+                trackerProcess.EnableRaisingEvents = true;
-                    trackerProcess.CancelOutputRead();
+                trackerProcess.OutputDataReceived += new DataReceivedEventHandler(delegate(object sender, DataReceivedEventArgs e) {
-                    trackerProcess.CancelErrorRead();
+                    trackerSB.Append(e.Data);
-                } catch {}
+                    trackerSB.Append("\n");
+                });
+                trackerProcess.ErrorDataReceived += new DataReceivedEventHandler(delegate(object sender, DataReceivedEventArgs e) {
+                    trackerSB.Append(e.Data);
+                    trackerSB.Append("\n");
+                });
+            }
+            trackerProcess.Start();
+            job.AddProcess(trackerProcess.Handle);
+            if (!dontPrintNow) {
+                trackerProcess.BeginOutputReadLine();
+                trackerProcess.BeginErrorReadLine();
             }
-        }
 
-        trackerSB = new StringBuilder();
+            return !trackerProcess.HasExited;
-        trackerProcess = new Process();
+        } else {
-        trackerProcess.StartInfo = processStartInfo;
+            string dir = Path.GetDirectoryName(exePath);
-        dontPrintNow = dontPrint;
+            string outputLog = Application.persistentDataPath + "/" + pinvokeStdOut;
-        if (!dontPrintNow) {
+            string errorLog = Application.persistentDataPath + "/" + pinvokeStdErr;
-            trackerProcess.EnableRaisingEvents = true;
+            OpenSeeProcessInterface.Start(exePath, "facetracker " + argumentString, dir, true, outputLog, errorLog, out processHandle, out processStdOut, out processStdErr);
-            trackerProcess.OutputDataReceived += new DataReceivedEventHandler(delegate(object sender, DataReceivedEventArgs e) {
+            if (processHandle != System.IntPtr.Zero) {
-                trackerSB.Append(e.Data);
+                job.AddProcess(processHandle);
-                trackerSB.Append("\n");
+                return OpenSeeProcessInterface.Alive(processHandle);
-            });
+            } else
-            trackerProcess.ErrorDataReceived += new DataReceivedEventHandler(delegate(object sender, DataReceivedEventArgs e) {
+                return false;
-                trackerSB.Append(e.Data);
-                trackerSB.Append("\n");
-            });
         }
-        trackerProcess.Start();
-        job.AddProcess(trackerProcess.Handle);
-        if (!dontPrintNow) {
-            trackerProcess.BeginOutputReadLine();
-            trackerProcess.BeginErrorReadLine();
-        }
-
-        return !trackerProcess.HasExited;
     }
     
     public void StopTracker() {
+        if (processHandle != System.IntPtr.Zero) {
+            if (OpenSeeProcessInterface.Alive(processHandle))
+                OpenSeeProcessInterface.TerminateProcess(processHandle, 0);
+            OpenSeeProcessInterface.CloseHandle(processHandle);
+            processHandle = System.IntPtr.Zero;
+        }
+        if (processStdOut != System.IntPtr.Zero) {
+            OpenSeeProcessInterface.CloseHandle(processStdOut);
+            processStdOut = System.IntPtr.Zero;
+        }
+        if (processStdErr != System.IntPtr.Zero) {
+            OpenSeeProcessInterface.CloseHandle(processStdErr);
+            processStdErr = System.IntPtr.Zero;
+        }
         if (trackerProcess != null && !trackerProcess.HasExited) {
             trackerProcess.CloseMainWindow();
             trackerProcess.Close();
@@ -246,11 +432,16 @@ public class OpenSeeLauncher : MonoBehaviour {
     }
     
     public void Update() {
-        if (trackerProcess != null)
+        if (processHandle != System.IntPtr.Zero) {
-            trackerAlive = !trackerProcess.HasExited;
+            trackerAlive = OpenSeeProcessInterface.Alive(processHandle);
-        else {
-            trackerAlive = false;
             return;
+        } else {
+            if (trackerProcess != null)
+                trackerAlive = !trackerProcess.HasExited;
+            else {
+                trackerAlive = false;
+                return;
+            }
         }
         if (dontPrintNow || trackerSB == null)
             return;

Unity/OpenSeeLauncher.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: beae834477f91834691fb7248fddacf6
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/OpenSeeProcessInterface.cs

@@ -0,0 +1,364 @@
+using System;
+using System.ComponentModel;
+using System.Diagnostics;
+using System.Runtime.InteropServices;
+
+namespace OpenSee {
+
+// Based on https://github.com/josh4364/IL2cppStartProcess/blob/master/StartExternalProcess.cs
+
+public class OpenSeeProcessInterface {
+    #region DllImport
+    [DllImport("kernel32.dll", SetLastError=true, CharSet=CharSet.Auto)]
+    private static extern IntPtr CreateFile(
+        string lpFileName,
+        EFileAccess dwDesiredAccess,
+        EFileShare dwShareMode,
+        SECURITY_ATTRIBUTES lpSecurityAttributes,
+        ECreationDisposition dwCreationDisposition,
+        EFileAttributes dwFlagsAndAttributes,
+        IntPtr hTemplateFile
+    );
+    
+    [StructLayout(LayoutKind.Sequential)]
+    private struct SECURITY_ATTRIBUTES
+    {
+        public int nLength;
+        public IntPtr lpSecurityDescriptor;
+        public int bInheritHandle;
+    }
+
+    [Flags]
+    enum EFileAccess : uint {
+        //
+        // Standart Section
+        //
+
+        AccessSystemSecurity = 0x1000000,   // AccessSystemAcl access type
+        MaximumAllowed = 0x2000000,     // MaximumAllowed access type
+
+        Delete = 0x10000,
+        ReadControl = 0x20000,
+        WriteDAC = 0x40000,
+        WriteOwner = 0x80000,
+        Synchronize = 0x100000,
+
+        StandardRightsRequired = 0xF0000,
+        StandardRightsRead = ReadControl,
+        StandardRightsWrite = ReadControl,
+        StandardRightsExecute = ReadControl,
+        StandardRightsAll = 0x1F0000,
+        SpecificRightsAll = 0xFFFF,
+
+        FILE_READ_DATA = 0x0001,        // file & pipe
+        FILE_LIST_DIRECTORY = 0x0001,       // directory
+        FILE_WRITE_DATA = 0x0002,       // file & pipe
+        FILE_ADD_FILE = 0x0002,         // directory
+        FILE_APPEND_DATA = 0x0004,      // file
+        FILE_ADD_SUBDIRECTORY = 0x0004,     // directory
+        FILE_CREATE_PIPE_INSTANCE = 0x0004, // named pipe
+        FILE_READ_EA = 0x0008,          // file & directory
+        FILE_WRITE_EA = 0x0010,         // file & directory
+        FILE_EXECUTE = 0x0020,          // file
+        FILE_TRAVERSE = 0x0020,         // directory
+        FILE_DELETE_CHILD = 0x0040,     // directory
+        FILE_READ_ATTRIBUTES = 0x0080,      // all
+        FILE_WRITE_ATTRIBUTES = 0x0100,     // all
+
+        //
+        // Generic Section
+        //
+
+        GenericRead = 0x80000000,
+        GenericWrite = 0x40000000,
+        GenericExecute = 0x20000000,
+        GenericAll = 0x10000000,
+
+        SPECIFIC_RIGHTS_ALL = 0x00FFFF,
+        FILE_ALL_ACCESS =
+            StandardRightsRequired |
+            Synchronize |
+            0x1FF,
+
+        FILE_GENERIC_READ =
+            StandardRightsRead |
+            FILE_READ_DATA |
+            FILE_READ_ATTRIBUTES |
+            FILE_READ_EA |
+            Synchronize,
+
+        FILE_GENERIC_WRITE =
+            StandardRightsWrite |
+            FILE_WRITE_DATA |
+            FILE_WRITE_ATTRIBUTES |
+            FILE_WRITE_EA |
+            FILE_APPEND_DATA |
+            Synchronize,
+
+        FILE_GENERIC_EXECUTE =
+            StandardRightsExecute |
+            FILE_READ_ATTRIBUTES |
+            FILE_EXECUTE |
+            Synchronize
+    }
+
+    [Flags]
+    public enum EFileShare : uint
+    {
+        /// <summary>
+        ///
+        /// </summary>
+        None = 0x00000000,
+        /// <summary>
+        /// Enables subsequent open operations on an object to request read access.
+        /// Otherwise, other processes cannot open the object if they request read access.
+        /// If this flag is not specified, but the object has been opened for read access, the function fails.
+        /// </summary>
+        Read = 0x00000001,
+        /// <summary>
+        /// Enables subsequent open operations on an object to request write access.
+        /// Otherwise, other processes cannot open the object if they request write access.
+        /// If this flag is not specified, but the object has been opened for write access, the function fails.
+        /// </summary>
+        Write = 0x00000002,
+        /// <summary>
+        /// Enables subsequent open operations on an object to request delete access.
+        /// Otherwise, other processes cannot open the object if they request delete access.
+        /// If this flag is not specified, but the object has been opened for delete access, the function fails.
+        /// </summary>
+        Delete = 0x00000004
+    }
+
+    public enum ECreationDisposition : uint
+    {
+        /// <summary>
+        /// Creates a new file. The function fails if a specified file exists.
+        /// </summary>
+        New = 1,
+        /// <summary>
+        /// Creates a new file, always.
+        /// If a file exists, the function overwrites the file, clears the existing attributes, combines the specified file attributes,
+        /// and flags with FILE_ATTRIBUTE_ARCHIVE, but does not set the security descriptor that the SECURITY_ATTRIBUTES structure specifies.
+        /// </summary>
+        CreateAlways = 2,
+        /// <summary>
+        /// Opens a file. The function fails if the file does not exist.
+        /// </summary>
+        OpenExisting = 3,
+        /// <summary>
+        /// Opens a file, always.
+        /// If a file does not exist, the function creates a file as if dwCreationDisposition is CREATE_NEW.
+        /// </summary>
+        OpenAlways = 4,
+        /// <summary>
+        /// Opens a file and truncates it so that its size is 0 (zero) bytes. The function fails if the file does not exist.
+        /// The calling process must open the file with the GENERIC_WRITE access right.
+        /// </summary>
+        TruncateExisting = 5
+    }
+
+    [Flags]
+    public enum EFileAttributes : uint
+    {
+        Readonly         = 0x00000001,
+        Hidden           = 0x00000002,
+        System           = 0x00000004,
+        Directory        = 0x00000010,
+        Archive          = 0x00000020,
+        Device           = 0x00000040,
+        Normal           = 0x00000080,
+        Temporary        = 0x00000100,
+        SparseFile       = 0x00000200,
+        ReparsePoint     = 0x00000400,
+        Compressed       = 0x00000800,
+        Offline          = 0x00001000,
+        NotContentIndexed= 0x00002000,
+        Encrypted        = 0x00004000,
+        Write_Through    = 0x80000000,
+        Overlapped       = 0x40000000,
+        NoBuffering      = 0x20000000,
+        RandomAccess     = 0x10000000,
+        SequentialScan   = 0x08000000,
+        DeleteOnClose    = 0x04000000,
+        BackupSemantics  = 0x02000000,
+        PosixSemantics   = 0x01000000,
+        OpenReparsePoint = 0x00200000,
+        OpenNoRecall     = 0x00100000,
+        FirstPipeInstance= 0x00080000
+    }
+
+    
+    [DllImport("kernel32.dll", SetLastError=true)]
+    private static extern UInt32 WaitForSingleObject(IntPtr hHandle, UInt32 dwMilliseconds);
+    const UInt32 INFINITE = 0xFFFFFFFF;
+    const UInt32 WAIT_ABANDONED = 0x00000080;
+    const UInt32 WAIT_OBJECT_0 = 0x00000000;
+    const UInt32 WAIT_TIMEOUT = 0x00000102;
+    
+    [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
+    [return: MarshalAs(UnmanagedType.Bool)]
+    private static extern bool CreateProcessW(
+        string lpApplicationName,
+        [In] string lpCommandLine,
+        IntPtr procSecAttrs,
+        IntPtr threadSecAttrs,
+        bool bInheritHandles,
+        ProcessCreationFlags dwCreationFlags,
+        IntPtr lpEnvironment,
+        string lpCurrentDirectory,
+        ref STARTUPINFO lpStartupInfo,
+        ref PROCESS_INFORMATION lpProcessInformation
+    );
+
+    [DllImport("kernel32.dll", SetLastError = true)]
+    [return: MarshalAs(UnmanagedType.Bool)]
+    public static extern bool CloseHandle(IntPtr hObject);
+
+    [DllImport("kernel32.dll", SetLastError = true)]
+    [return: MarshalAs(UnmanagedType.Bool)]
+    public static extern bool TerminateProcess(IntPtr processHandle, uint exitCode);
+
+    [DllImport("kernel32.dll", SetLastError = true)]
+    private static extern IntPtr OpenProcess(ProcessAccessRights access, bool inherit, uint processId);
+
+    [Flags]
+    private enum ProcessAccessRights : uint
+    {
+        PROCESS_CREATE_PROCESS = 0x0080, //  Required to create a process.
+        PROCESS_CREATE_THREAD = 0x0002, //  Required to create a thread.
+        PROCESS_DUP_HANDLE = 0x0040, // Required to duplicate a handle using DuplicateHandle.
+        PROCESS_QUERY_INFORMATION = 0x0400, //  Required to retrieve certain information about a process, such as its token, exit code, and priority class (see OpenProcessToken, GetExitCodeProcess, GetPriorityClass, and IsProcessInJob).
+        PROCESS_QUERY_LIMITED_INFORMATION = 0x1000, //  Required to retrieve certain information about a process (see QueryFullProcessImageName). A handle that has the PROCESS_QUERY_INFORMATION access right is automatically granted PROCESS_QUERY_LIMITED_INFORMATION. Windows Server 2003 and Windows XP/2000:  This access right is not supported.
+        PROCESS_SET_INFORMATION = 0x0200, //    Required to set certain information about a process, such as its priority class (see SetPriorityClass).
+        PROCESS_SET_QUOTA = 0x0100, //  Required to set memory limits using SetProcessWorkingSetSize.
+        PROCESS_SUSPEND_RESUME = 0x0800, // Required to suspend or resume a process.
+        PROCESS_TERMINATE = 0x0001, //  Required to terminate a process using TerminateProcess.
+        PROCESS_VM_OPERATION = 0x0008, //   Required to perform an operation on the address space of a process (see VirtualProtectEx and WriteProcessMemory).
+        PROCESS_VM_READ = 0x0010, //    Required to read memory in a process using ReadProcessMemory.
+        PROCESS_VM_WRITE = 0x0020, //   Required to write to memory in a process using WriteProcessMemory.
+        DELETE = 0x00010000, // Required to delete the object.
+        READ_CONTROL = 0x00020000, //   Required to read information in the security descriptor for the object, not including the information in the SACL. To read or write the SACL, you must request the ACCESS_SYSTEM_SECURITY access right. For more information, see SACL Access Right.
+        SYNCHRONIZE = 0x00100000, //    The right to use the object for synchronization. This enables a thread to wait until the object is in the signaled state.
+        WRITE_DAC = 0x00040000, //  Required to modify the DACL in the security descriptor for the object.
+        WRITE_OWNER = 0x00080000, //    Required to change the owner in the security descriptor for the object.
+        STANDARD_RIGHTS_REQUIRED = 0x000f0000,
+        PROCESS_ALL_ACCESS = STANDARD_RIGHTS_REQUIRED | SYNCHRONIZE | 0xFFF //    All possible access rights for a process object.
+    }
+
+    [StructLayout(LayoutKind.Sequential)]
+    private struct PROCESS_INFORMATION
+    {
+        internal IntPtr hProcess;
+        internal IntPtr hThread;
+        internal uint dwProcessId;
+        internal uint dwThreadId;
+    }
+
+    [StructLayout(LayoutKind.Sequential)]
+    private struct STARTUPINFO
+    {
+        internal uint cb;
+        internal IntPtr lpReserved;
+        internal IntPtr lpDesktop;
+        internal IntPtr lpTitle;
+        internal uint dwX;
+        internal uint dwY;
+        internal uint dwXSize;
+        internal uint dwYSize;
+        internal uint dwXCountChars;
+        internal uint dwYCountChars;
+        internal uint dwFillAttribute;
+        internal uint dwFlags;
+        internal ushort wShowWindow;
+        internal ushort cbReserved2;
+        internal IntPtr lpReserved2;
+        internal IntPtr hStdInput;
+        internal IntPtr hStdOutput;
+        internal IntPtr hStdError;
+    }
+
+    [Flags]
+    private enum ProcessCreationFlags : uint
+    {
+        NONE = 0,
+        CREATE_BREAKAWAY_FROM_JOB = 0x01000000,
+        CREATE_DEFAULT_ERROR_MODE = 0x04000000,
+        CREATE_NEW_CONSOLE = 0x00000010,
+        CREATE_NEW_PROCESS_GROUP = 0x00000200,
+        CREATE_NO_WINDOW = 0x08000000,
+        CREATE_PROTECTED_PROCESS = 0x00040000,
+        CREATE_PRESERVE_CODE_AUTHZ_LEVEL = 0x02000000,
+        CREATE_SECURE_PROCESS = 0x00400000,
+        CREATE_SEPARATE_WOW_VDM = 0x00000800,
+        CREATE_SHARED_WOW_VDM = 0x00001000,
+        CREATE_SUSPENDED = 0x00000004,
+        CREATE_UNICODE_ENVIRONMENT = 0x00000400,
+        DEBUG_ONLY_THIS_PROCESS = 0x00000002,
+        DEBUG_PROCESS = 0x00000001,
+        DETACHED_PROCESS = 0x00000008,
+        EXTENDED_STARTUPINFO_PRESENT = 0x00080000,
+        INHERIT_PARENT_AFFINITY = 0x00010000
+    }
+    #endregion
+    
+    public static uint Start(string exePath, string commandline, string dir, bool hidden, string outputLog, string errorLog, out IntPtr handle, out IntPtr handleStdOut, out IntPtr handleStdErr)
+    {
+        SECURITY_ATTRIBUTES sa = new SECURITY_ATTRIBUTES();
+        sa.nLength = Marshal.SizeOf<SECURITY_ATTRIBUTES>();
+        sa.lpSecurityDescriptor = IntPtr.Zero;
+        sa.bInheritHandle = 1;
+        
+        handle = IntPtr.Zero;
+        handleStdOut = IntPtr.Zero;
+        handleStdErr = IntPtr.Zero;
+        if (outputLog != null)
+            handleStdOut = CreateFile(outputLog, EFileAccess.FILE_WRITE_DATA, EFileShare.Read | EFileShare.Write, sa, ECreationDisposition.CreateAlways, EFileAttributes.Normal, IntPtr.Zero);
+        if (errorLog != null)
+            handleStdErr = CreateFile(errorLog, EFileAccess.FILE_WRITE_DATA, EFileShare.Read | EFileShare.Write, sa, ECreationDisposition.CreateAlways, EFileAttributes.Normal, IntPtr.Zero);
+        
+        ProcessCreationFlags flags = hidden ? ProcessCreationFlags.CREATE_NO_WINDOW : ProcessCreationFlags.NONE;
+        STARTUPINFO startupinfo = new STARTUPINFO
+        {
+            cb = (uint)Marshal.SizeOf<STARTUPINFO>(),
+            dwFlags = 0x00000100,
+            hStdInput = IntPtr.Zero,
+            hStdOutput = handleStdOut,
+            hStdError = handleStdErr
+        };
+        PROCESS_INFORMATION processinfo = new PROCESS_INFORMATION();
+        if (!CreateProcessW(exePath, commandline, IntPtr.Zero, IntPtr.Zero, true, flags, IntPtr.Zero, dir, ref startupinfo, ref processinfo))
+        {
+            throw new Win32Exception();
+        }
+        handle = OpenProcess(ProcessAccessRights.PROCESS_ALL_ACCESS, false, processinfo.dwProcessId);
+
+        return processinfo.dwProcessId;
+    }
+    
+    public static bool Alive(IntPtr handle) {
+        uint ret = WaitForSingleObject(handle, 0);
+        return ret == WAIT_TIMEOUT;
+    }
+
+    public static int KillProcess(uint pid)
+    {
+        IntPtr handle = OpenProcess(ProcessAccessRights.PROCESS_ALL_ACCESS, false, pid);
+
+        if (handle == IntPtr.Zero)
+        {
+            return -1;
+        }
+        if (!TerminateProcess(handle, 0))
+        {
+            throw new Win32Exception();
+        }
+        if (!CloseHandle(handle))
+        {
+            throw new Win32Exception();
+        }
+
+        return 0;
+    }
+}
+
+}

Unity/OpenSeeProcessInterface.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 8130804d2fedd6d47a6cf0a447058d08
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/OpenSeeShowPoints.cs

@@ -6,60 +6,143 @@ namespace OpenSee {
 
 public class OpenSeeShowPoints : MonoBehaviour {
     public OpenSee openSee = null;
+    public int faceId = 0;
+    public bool only30Points = false;
     public bool show3DPoints = true;
     public bool applyTranslation = false;
     public bool applyRotation = false;
+    [Range(0, 1)]
     public float minConfidence = 0.20f;
-    private OpenSee.OpenSeeData[] openSeeData;
+    public bool showGaze = true;
+    public Material material;
+    public bool showLines = false;
+    public float lineWidth = 0.01f;
+    public Material lineMaterial;
+    public bool receiveShadows = false;
+    
+    private OpenSee.OpenSeeData openSeeData;
     private GameObject[] gameObjects;
+    private LineRenderer[] lineRenderers;
     private GameObject centerBall;
     private double updated = 0.0;
+    private int total = 70;
+    
+    private int[] lines = new int[]{/* Contour */ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, -1, /* Eyebrows */ 18, 19, 20, 21, -1, 23, 24, 25, 26, -1, /* Nose */ 28, 29, 30, 33, 32, 33, 34, 35, -1, /* Eye */ 37, 38, 39, 40, 41, 36, /* Eye */ 43, 44, 45, 46, 47, 42, /* Mouth */ 49, 50, 51, 52, 62, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 58, 58, 62};
+    
+    private HashSet<int> pt30Set = new HashSet<int> { 0, 2, 5, 8, 11, 14, 16, 17, 19, 21, 22, 24, 26, 27, 30, 33, 36, 37, 39, 40, 42, 43, 45, 46, 50, 55, 58, 60, 62, 64 };
+    private int[] linesPt30Set = new int[]{/* Contour */ 2, -1, 5, -1, -1, 8, -1, -1, 11, -1, -1, 14, -1, -1, 16, -1, -1, /* Eyebrows */ 19, -1, 21, -1, -1, 24, -1, 26, -1, -1, /* Nose */ 30, -1, -1, 33, -1, -1, -1, -1, -1, /* Eye */ 37, 39, -1, 40, 36, -1, /* Eye */ 43, 45, -1, 46, 42, -1, /* Mouth */ -1, -1, 62, -1, -1, -1, -1, 58, -1, -1, 60, -1, 62, -1, 64, -1, 58, -1, 58, 62};
 
 	void Start () {
         if (openSee == null) {
             openSee = GetComponent<OpenSee>();
         }
-        gameObjects = new GameObject[68];
+        gameObjects = new GameObject[70];
-        for (int i = 0; i < 68; i++) {
+        lineRenderers = new LineRenderer[68];
+        if (lineMaterial == null)
+            showLines = false;
+        if (!showGaze)
+            total = 66;
+        for (int i = 0; i < total; i++) {
             gameObjects[i] = GameObject.CreatePrimitive(PrimitiveType.Sphere);
             gameObjects[i].name = "Point " + (i + 1);
+            if (material != null)
+                gameObjects[i].GetComponent<Renderer>().material = material;
+            gameObjects[i].GetComponent<Renderer>().receiveShadows = receiveShadows;
+            gameObjects[i].layer = gameObject.layer;
             gameObjects[i].transform.SetParent(transform);
             gameObjects[i].transform.localScale = new Vector3(0.025f, 0.025f, 0.025f);
+            if (i >= 68) {
+                GameObject cylinder = GameObject.CreatePrimitive(PrimitiveType.Cylinder);
+                if (material != null)
+                    cylinder.GetComponent<Renderer>().material = material;
+                cylinder.GetComponent<Renderer>().receiveShadows = receiveShadows;
+                cylinder.layer = gameObject.layer;
+                cylinder.transform.SetParent(gameObjects[i].transform);
+                cylinder.transform.localEulerAngles = new Vector3(90f, 0f, 0f);
+                cylinder.transform.localPosition = new Vector3(0f, 0f, -4f);
+                cylinder.transform.localScale = new Vector3(1f, 4f, 1f);
+            }
+            if (only30Points && i < 66 && !pt30Set.Contains(i))
+                gameObjects[i].SetActive(false);
+        }
+        for (int i = 0; i < 68; i++) {
+                if (i == 66) {
+                    GameObject lineGameObject = new GameObject("LineGameObject");
+                    if (only30Points)
+                        lineGameObject.transform.SetParent(gameObjects[50].transform);
+                    else
+                        lineGameObject.transform.SetParent(gameObjects[48].transform);
+                    lineGameObject.layer = gameObject.layer;
+                    lineRenderers[i] = lineGameObject.AddComponent(typeof(LineRenderer)) as LineRenderer;
+                } else if (i == 67) {
+                    GameObject lineGameObject = new GameObject("LineGameObject");
+                    if (only30Points)
+                        lineGameObject.transform.SetParent(gameObjects[55].transform);
+                    else
+                        lineGameObject.transform.SetParent(gameObjects[53].transform);
+                    lineGameObject.layer = gameObject.layer;
+                    lineRenderers[i] = lineGameObject.AddComponent(typeof(LineRenderer)) as LineRenderer;
+                }
+                else
+                    lineRenderers[i] = gameObjects[i].AddComponent(typeof(LineRenderer)) as LineRenderer;
+                lineRenderers[i].useWorldSpace = true;
+                lineRenderers[i].generateLightingData = true;
+                lineRenderers[i].material = lineMaterial;
+                lineRenderers[i].receiveShadows = receiveShadows;
+                lineRenderers[i].widthMultiplier = lineWidth;
+                lineRenderers[i].positionCount = 2;
         }
         centerBall = GameObject.CreatePrimitive(PrimitiveType.Sphere);
         centerBall.name = "Center";
         centerBall.transform.SetParent(transform);
         centerBall.transform.localScale = new Vector3(0.1f, 0.1f, 0.1f);
 	}
-    
+
 	void Update () {
         if (!openSee)
             return;
-        openSeeData = openSee.trackingData;
+        /*openSeeData = openSee.trackingData;
         if (openSeeData == null || openSeeData.Length < 1)
+            return;*/
+        openSeeData = openSee.GetOpenSeeData(faceId);
+        if (openSeeData == null || (show3DPoints && openSeeData.fit3DError > openSee.maxFit3DError))
             return;
-        if (openSeeData[0].time > updated) {
+        if (openSeeData.time > updated) {
-            updated = openSeeData[0].time;
+            updated = openSeeData.time;
         } else {
             return;
         }
         if (show3DPoints) {
             centerBall.gameObject.SetActive(false);
-            for (int i = 0; i < 66; i++) {
+            for (int i = 0; i < total; i++) {
-                if (openSeeData[0].got3DPoints && openSeeData[0].confidence[i] > minConfidence) {
+                if (openSeeData.got3DPoints && (i >= 68 || openSeeData.confidence[i] > minConfidence)) {
                     Renderer renderer = gameObjects[i].GetComponent<Renderer>();
-                    renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData[0].confidence[i]));
+                    Vector3 pt = openSeeData.points3D[i];
-                    gameObjects[i].transform.localPosition = openSeeData[0].points3D[i];
+                    pt.x = -pt.x;
+                    gameObjects[i].transform.localPosition = pt;
+                    if (i < 68)
+                        renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i]));
+                    else {
+                        if (i == 68)
+                            gameObjects[i].transform.localRotation = openSeeData.rightGaze;
+                        else
+                            gameObjects[i].transform.localRotation = openSeeData.leftGaze;
+                    }
                 } else {
                     Renderer renderer = gameObjects[i].GetComponent<Renderer>();
                     renderer.material.SetColor("_Color", Color.cyan);
                 }
             }
             if (applyTranslation) {
-                transform.position = openSeeData[0].translation;
+                Vector3 v = openSeeData.translation;
+                v.x = -v.x;
+                v.z = -v.z;
+                transform.localPosition = v;
             }
             if (applyRotation) {
-                transform.eulerAngles = openSeeData[0].rotation;
+                Quaternion offset = Quaternion.Euler(0f, 0f, -90f);
+                Quaternion convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w) * offset;
+                transform.localRotation = convertedQuaternion;
             }
         } else {
             centerBall.gameObject.SetActive(false);
@@ -69,8 +152,8 @@ public class OpenSeeShowPoints : MonoBehaviour {
             float maxX = -1.0f;
             float maxY = -1.0f;
             for (int i = 0; i < 66; i++) {
-                float x = openSeeData[0].points[i].x;
+                float x = openSeeData.points[i].x;
-                float y = -openSeeData[0].points[i].y;
+                float y = -openSeeData.points[i].y;
                 if (minX > x) {
                     minX = x;
                 }
@@ -91,12 +174,13 @@ public class OpenSeeShowPoints : MonoBehaviour {
             center.y = center.y / (maxX - minX);
             center.z = 0.5f;
             centerBall.transform.localPosition = center;
-            
+
             for (int i = 0; i < 66; i++) {
                 Renderer renderer = gameObjects[i].GetComponent<Renderer>();
-                renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData[0].confidence[i]));
+                renderer.receiveShadows = receiveShadows;
-                float x = openSeeData[0].points[i].x;
+                renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i]));
-                float y = -openSeeData[0].points[i].y;
+                float x = openSeeData.points[i].x;
+                float y = -openSeeData.points[i].y;
                 Vector3 position = new Vector3(x, y, 0.0f);
                 position = position - new Vector3(minX, minY, 0.0f);
                 position.x = position.x / (maxX - minX);
@@ -104,6 +188,37 @@ public class OpenSeeShowPoints : MonoBehaviour {
                 gameObjects[i].transform.localPosition = position;
             }
         }
+        for (int i = 0; i < 68; i++) {
+            if ((!only30Points && lines[i] == -1) || (only30Points && linesPt30Set[i] == -1))
+                continue;
+            if (!showLines || lineMaterial == null) {
+                lineRenderers[i].enabled = false;
+            } else {
+                int a = i;
+                int b = lines[i];
+                if (only30Points) {
+                    b = linesPt30Set[i];
+                    if (i == 66)
+                        a = 50;
+                    if (i == 67)
+                        a = 55;
+                } else {
+                    if (i == 66)
+                        a = 48;
+                    if (i == 67)
+                        a = 53;
+                }
+                Color color = Color.Lerp(Color.red, Color.green, Mathf.Lerp(0.5f, openSeeData.confidence[a], openSeeData.confidence[b]));
+                lineRenderers[i].enabled = true;
+                lineRenderers[i].widthMultiplier = lineWidth;
+                lineRenderers[i].receiveShadows = receiveShadows;
+                lineRenderers[i].material.SetColor("_Color", color);
+                lineRenderers[i].startColor = color;
+                lineRenderers[i].endColor = color;
+                lineRenderers[i].SetPosition(0, gameObjects[a].transform.position);
+                lineRenderers[i].SetPosition(1, gameObjects[b].transform.position);
+            }
+        }
 	}
 }
 

Unity/OpenSeeShowPoints.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 9c0b8bb971ed0e14288631f4274a3807
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/Plugins.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: f1dfc5f1c2e6f294e8191828595882b1
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/Plugins/SVMModel.cs

@@ -0,0 +1,285 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Runtime.InteropServices;
+using System.Runtime.Serialization;
+using System.Runtime.Serialization.Formatters.Binary;
+using System.Text;
+using UnityEngine;
+
+namespace OpenSee {
+
+public class SVMModel {
+	#region DllImport
+	[DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl, EntryPoint = "trainModel")]
+	private static extern System.IntPtr trainModelDll(float[] features, float[] labels, float[] weights, int rows, int cols, int classes, int probability, float C);
+    protected virtual System.IntPtr trainModel(float[] features, float[] labels, float[] weights, int rows, int cols, int classes, int probability, float C) {
+        Debug.Log("Training LibSVM");
+        return trainModelDll(features, labels, weights, rows, cols, classes, probability, C);
+    }
+
+	[DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl, EntryPoint = "predict")]
+	private static extern void predictDll(System.IntPtr model, float[] features, [Out] float[] predictions, [Out] double[] probabilities, int rows);
+    protected virtual void predict(System.IntPtr model, float[] features, [Out] float[] predictions, [Out] double[] probabilities, int rows) {
+        predictDll(model, features, predictions, probabilities, rows);
+    }
+
+	[DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi, EntryPoint = "loadModel")]
+	private static extern System.IntPtr loadModelDll([MarshalAs(UnmanagedType.LPStr)]string filename, int cols, int classes, double[] means, double[] sdevs);
+    protected virtual System.IntPtr loadModel([MarshalAs(UnmanagedType.LPStr)]string filename, int cols, int classes, double[] means, double[] sdevs) {
+        return loadModelDll(filename, cols, classes, means, sdevs);
+    }
+
+	[DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi, EntryPoint = "loadModelString")]
+	private static extern System.IntPtr loadModelStringDll([MarshalAs(UnmanagedType.LPStr)]string modelString, int cols, int classes, double[] means, double[] sdevs);
+    protected virtual System.IntPtr loadModelString([MarshalAs(UnmanagedType.LPStr)]string modelString, int cols, int classes, double[] means, double[] sdevs) {
+        Debug.Log("Loading LibSVM");
+        return loadModelStringDll(modelString, cols, classes, means, sdevs);
+    }
+
+	[DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl, EntryPoint = "getScales")]
+	private static extern void getScalesDll(System.IntPtr model, [Out] double[] means, [Out] double[] sdevs);
+    protected virtual void getScales(System.IntPtr model, [Out] double[] means, [Out] double[] sdevs) {
+        getScalesDll(model, means, sdevs);
+    }
+
+	[DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi, EntryPoint = "saveModel")]
+	private static extern int saveModelDll(System.IntPtr model, [MarshalAs(UnmanagedType.LPStr)]string filename);
+    protected virtual int saveModel(System.IntPtr model, [MarshalAs(UnmanagedType.LPStr)]string filename) {
+        return saveModelDll(model, filename);
+    }
+
+	[DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi, EntryPoint = "saveModelString")]
+	private static extern string saveModelStringDll(System.IntPtr model);
+    protected virtual string saveModelString(System.IntPtr model) {
+        return saveModelStringDll(model);
+    }
+
+	[DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl, EntryPoint = "destroyModel")]
+	private static extern void destroyModelDll(System.IntPtr model);
+    protected virtual void destroyModel(System.IntPtr model) {
+        destroyModelDll(model);
+    }
+	#endregion
+
+    private bool haveModel = false;
+    private System.IntPtr model;
+    private int cols;
+    private int maxClasses;
+
+    [Serializable]
+    private class SVMModelRepresentation {
+        public string modelString;
+        public int cols;
+        public int maxClasses;
+        public double[] means;
+        public double[] sdevs;
+    }
+
+    public System.IntPtr LoadSerialized(byte[] modelBytes, out int cols, out int maxClasses, bool compress) {
+        IFormatter formatter = new BinaryFormatter();
+        MemoryStream memoryStream = new MemoryStream(modelBytes);
+        memoryStream.Position = 0;
+        SVMModelRepresentation smr;
+        if (compress) {
+            using (GZipStream gzipStream = new GZipStream(memoryStream, CompressionMode.Decompress)) {
+                smr = formatter.Deserialize(gzipStream) as SVMModelRepresentation;
+            }
+        } else {
+            smr = formatter.Deserialize(memoryStream) as SVMModelRepresentation;
+        }
+        cols = smr.cols;
+        maxClasses = smr.maxClasses;
+        System.IntPtr model = loadModelString(smr.modelString, smr.cols, smr.maxClasses, smr.means, smr.sdevs);
+        return model;
+    }
+
+    public byte[] ToSerialized(System.IntPtr model, int cols, int maxClasses, bool compress) {
+        SVMModelRepresentation smr = new SVMModelRepresentation();
+        smr.cols = cols;
+        smr.maxClasses = maxClasses;
+        smr.means = new double[cols];
+        smr.sdevs = new double[cols];
+        getScales(model, smr.means, smr.sdevs);
+        smr.modelString = saveModelString(model);
+
+        IFormatter formatter = new BinaryFormatter();
+        MemoryStream memoryStream = new MemoryStream();
+        if (compress) {
+            using (GZipStream gzipStream = new GZipStream(memoryStream, CompressionMode.Compress)) {
+                formatter.Serialize(gzipStream, smr);
+                gzipStream.Flush();
+            }
+        } else {
+            formatter.Serialize(memoryStream, smr);
+        }
+        return memoryStream.ToArray();
+    }
+
+    public void DestroyModel() {
+        if (haveModel) {
+            destroyModel(model);
+            haveModel = false;
+        }
+    }
+
+    private void LoadModel(byte[] modelBytes, bool compress) {
+        DestroyModel();
+        model = LoadSerialized(modelBytes, out cols, out maxClasses, compress);
+        haveModel = true;
+    }
+
+    public void LoadModel(byte[] modelBytes) {
+        LoadModel(modelBytes, true);
+    }
+
+    private byte[] SaveModel(bool compress) {
+        if (!haveModel)
+            return null;
+        return ToSerialized(model, cols, maxClasses, compress);
+    }
+
+    public byte[] SaveModel() {
+        return SaveModel(true);
+    }
+
+    // features is a rows*cols long float array with rows written one after another. labels is a rows long float array with the corresponding classes as integral numbers from 0 to num_classes-1, where all classes have to exist. Training on more than 10000 rows is disabled.
+    public bool TrainModel(float[] features, float[] labels, float[] weights, int rows, int cols, int probability, float C) {
+        DestroyModel();
+        //if (rows > 10000)
+        //    return false;
+        if (cols < 1 || rows < 1 || features.Length < rows * cols || labels.Length < rows)
+            return false;
+
+        int max = -1;
+        HashSet<int> knownClasses = new HashSet<int>();
+        for (int i = 0; i < rows; i++) {
+            labels[i] = (float)Math.Round(labels[i]);
+            int l = (int)labels[i];
+            if (l > max)
+                max = l;
+            if (l < 0)
+                return false;
+            knownClasses.Add(l);
+        }
+        for (int i = 0; i <= max; i++) {
+            if (!knownClasses.Contains(i))
+                return false;
+        }
+
+        maxClasses = max + 1;
+        this.cols = cols;
+        float start = Time.realtimeSinceStartup;
+        model = trainModel(features, labels, weights, rows, cols, maxClasses, probability, C);
+        Debug.Log("Training took: " + (Time.realtimeSinceStartup - start).ToString());
+        haveModel = true;
+
+        return true;
+    }
+
+    // This function takes the same arguments as train, except for cols, which is not necessary. The diagonal are correctly classified entries. The first index is labels, the second index is classifications.
+    public int[,] ConfusionMatrix(float[] features, float[] labels, int rows, out float accuracy) {
+        accuracy = 0f;
+        if (!haveModel)
+            return null;
+        if (rows < 1 || features.Length < rows * cols || labels.Length < rows)
+            return null;
+        for (int i = 0; i < rows; i++) {
+            labels[i] = (float)Math.Round(labels[i]);
+            if (labels[i] < 0 || labels[i] >= maxClasses)
+                return null;
+        }
+        int[,] confusionMatrix = new int[maxClasses, maxClasses];
+        float[] predictions = new float[rows];
+        double[] probabilities = new double[rows * maxClasses];
+        predict(model, features, predictions, probabilities, rows);
+        for (int i = 0; i < rows; i++) {
+            predictions[i] = (float)Math.Round(predictions[i]);
+            if (predictions[i] >= 0 && predictions[i] < maxClasses) {
+                confusionMatrix[(int)labels[i], (int)predictions[i]]++;
+                if ((int)labels[i] == (int)predictions[i])
+                    accuracy++;
+            }
+        }
+        accuracy /= (float)rows;
+        return confusionMatrix;
+    }
+
+    static public string FormatMatrix(int[,] matrix) {
+        return FormatMatrix(matrix, null);
+    }
+
+    static public string FormatMatrix(int[,] matrix, string[] labels) {
+        StringBuilder sb = new StringBuilder();
+        int rowLength = matrix.GetLength(0);
+        int colLength = matrix.GetLength(1);
+        int maxLableLen = 5;
+
+        if (labels != null) {
+            if (labels.Length != rowLength || labels.Length != colLength)
+                labels = null;
+            else {
+                foreach (string label in labels)
+                    if (label.Length > maxLableLen)
+                        maxLableLen = label.Length;
+                sb.Append("pred:".PadLeft(maxLableLen));
+                sb.Append("     ");
+                foreach (string label in labels) {
+                    sb.Append(label.PadLeft(maxLableLen));
+                    sb.Append("     ");
+                }
+                sb.Append(Environment.NewLine);
+            }
+        }
+
+        for (int r = 0; r < rowLength; r++) {
+            if (labels != null) {
+                sb.Append(labels[r].PadLeft(maxLableLen));
+                sb.Append("     ");
+            }
+            for (int c = 0; c < colLength; c++) {
+                string n = matrix[r,c].ToString();
+                sb.Append(n.PadLeft(maxLableLen));
+                sb.Append("     ");
+            }
+            sb.Append(Environment.NewLine);
+        }
+        return sb.ToString();
+    }
+
+    // features is a rows*cols long float array with rows written one after another.
+    public float[] Predict(float[] features, out double[] probabilities, int rows) {
+        probabilities = null;
+        if (!haveModel)
+            return null;
+        if (rows < 1 || features.Length != rows * cols)
+            return null;
+        float[] predictions = new float[rows];
+        probabilities = new double[rows * maxClasses];
+        predict(model, features, predictions, probabilities, rows);
+        return predictions;
+    }
+
+    public float[] Predict(float[] features, int rows) {
+        double[] probabilities;
+        return Predict(features, out probabilities, rows);
+    }
+
+    public bool Ready() {
+        return haveModel;
+    }
+
+    public SVMModel() {
+    }
+
+    public SVMModel(byte[] modelBytes) {
+        LoadModel(modelBytes);
+    }
+
+    ~SVMModel() {
+        DestroyModel();
+    }
+}
+
+}

Unity/Plugins/SVMModel.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: a95b57bbe200b004dab9b21f0854082e
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/Plugins/SVMModel.dll.meta

@@ -0,0 +1,92 @@
+fileFormatVersion: 2
+guid: 930906668f1b67a42b2bb844bd098a26
+PluginImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  iconMap: {}
+  executionOrder: {}
+  isPreloaded: 0
+  isOverridable: 0
+  platformData:
+  - first:
+      '': Any
+    second:
+      enabled: 0
+      settings:
+        Exclude Editor: 0
+        Exclude Linux: 0
+        Exclude Linux64: 0
+        Exclude LinuxUniversal: 0
+        Exclude OSXUniversal: 0
+        Exclude WebGL: 0
+        Exclude Win: 1
+        Exclude Win64: 0
+  - first:
+      Any: 
+    second:
+      enabled: 1
+      settings: {}
+  - first:
+      Editor: Editor
+    second:
+      enabled: 1
+      settings:
+        CPU: AnyCPU
+        DefaultValueInitialized: true
+        OS: AnyOS
+  - first:
+      Facebook: Win
+    second:
+      enabled: 0
+      settings:
+        CPU: None
+  - first:
+      Facebook: Win64
+    second:
+      enabled: 0
+      settings:
+        CPU: AnyCPU
+  - first:
+      Standalone: Linux
+    second:
+      enabled: 1
+      settings:
+        CPU: x86
+  - first:
+      Standalone: Linux64
+    second:
+      enabled: 1
+      settings:
+        CPU: x86_64
+  - first:
+      Standalone: LinuxUniversal
+    second:
+      enabled: 1
+      settings:
+        CPU: AnyCPU
+  - first:
+      Standalone: OSXUniversal
+    second:
+      enabled: 1
+      settings:
+        CPU: AnyCPU
+  - first:
+      Standalone: Win
+    second:
+      enabled: 0
+      settings:
+        CPU: None
+  - first:
+      Standalone: Win64
+    second:
+      enabled: 1
+      settings:
+        CPU: AnyCPU
+  - first:
+      WebGL: WebGL
+    second:
+      enabled: 1
+      settings: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/Plugins/ThunderSVMModel.cs

@@ -0,0 +1,71 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Runtime.InteropServices;
+using System.Runtime.Serialization;
+using System.Runtime.Serialization.Formatters.Binary;
+using System.Text;
+using OpenSee;
+using UnityEngine;
+
+public class ThunderSVMModel : SVMModel {
+	#region DllImport
+	[DllImport("ThunderSVMWrapper", CallingConvention = CallingConvention.Cdecl, EntryPoint = "trainModel")]
+	private static extern System.IntPtr trainModelDll(float[] features, float[] labels, float[] weights, int rows, int cols, int classes, int probability, float C);
+    protected override System.IntPtr trainModel(float[] features, float[] labels, float[] weights, int rows, int cols, int classes, int probability, float C) {
+        Debug.Log("Training ThunderSVM");
+        return trainModelDll(features, labels, weights, rows, cols, classes, probability, C);
+    }
+
+	[DllImport("ThunderSVMWrapper", CallingConvention = CallingConvention.Cdecl, EntryPoint = "predict")]
+	private static extern void predictDll(System.IntPtr model, float[] features, [Out] float[] predictions, [Out] double[] probabilities, int rows);
+    protected override void predict(System.IntPtr model, float[] features, [Out] float[] predictions, [Out] double[] probabilities, int rows) {
+        predictDll(model, features, predictions, probabilities, rows);
+    }
+
+	[DllImport("ThunderSVMWrapper", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi, EntryPoint = "loadModel")]
+	private static extern System.IntPtr loadModelDll([MarshalAs(UnmanagedType.LPStr)]string filename, int cols, int classes, double[] means, double[] sdevs);
+    protected override System.IntPtr loadModel([MarshalAs(UnmanagedType.LPStr)]string filename, int cols, int classes, double[] means, double[] sdevs) {
+        return loadModelDll(filename, cols, classes, means, sdevs);
+    }
+
+	[DllImport("ThunderSVMWrapper", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi, EntryPoint = "loadModelString")]
+	private static extern System.IntPtr loadModelStringDll([MarshalAs(UnmanagedType.LPStr)]string modelString, int cols, int classes, double[] means, double[] sdevs);
+    protected override System.IntPtr loadModelString([MarshalAs(UnmanagedType.LPStr)]string modelString, int cols, int classes, double[] means, double[] sdevs) {
+        Debug.Log("Loading ThunderSVM");
+        return loadModelStringDll(modelString, cols, classes, means, sdevs);
+    }
+
+	[DllImport("ThunderSVMWrapper", CallingConvention = CallingConvention.Cdecl, EntryPoint = "getScales")]
+	private static extern void getScalesDll(System.IntPtr model, [Out] double[] means, [Out] double[] sdevs);
+    protected override void getScales(System.IntPtr model, [Out] double[] means, [Out] double[] sdevs) {
+        getScalesDll(model, means, sdevs);
+    }
+
+	[DllImport("ThunderSVMWrapper", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi, EntryPoint = "saveModel")]
+	private static extern int saveModelDll(System.IntPtr model, [MarshalAs(UnmanagedType.LPStr)]string filename);
+    protected override int saveModel(System.IntPtr model, [MarshalAs(UnmanagedType.LPStr)]string filename) {
+        return saveModelDll(model, filename);
+    }
+
+	[DllImport("ThunderSVMWrapper", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi, EntryPoint = "saveModelString")]
+	private static extern string saveModelStringDll(System.IntPtr model);
+    protected override string saveModelString(System.IntPtr model) {
+        return saveModelStringDll(model);
+    }
+
+	[DllImport("ThunderSVMWrapper", CallingConvention = CallingConvention.Cdecl, EntryPoint = "destroyModel")]
+	private static extern void destroyModelDll(System.IntPtr model);
+    protected override void destroyModel(System.IntPtr model) {
+        destroyModelDll(model);
+    }
+	#endregion
+    
+    public ThunderSVMModel() {
+    }
+
+    public ThunderSVMModel(byte[] modelBytes) {
+        LoadModel(modelBytes);
+    }
+}

Unity/Plugins/ThunderSVMWrapper.dll.meta

@@ -0,0 +1,63 @@
+fileFormatVersion: 2
+guid: 9c58dd9d1f0cf3140bd004d536712abb
+PluginImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  iconMap: {}
+  executionOrder: {}
+  defineConstraints: []
+  isPreloaded: 0
+  isOverridable: 0
+  isExplicitlyReferenced: 0
+  validateReferences: 1
+  platformData:
+  - first:
+      : Any
+    second:
+      enabled: 0
+      settings:
+        Exclude Editor: 0
+        Exclude Linux64: 0
+        Exclude OSXUniversal: 0
+        Exclude Win: 1
+        Exclude Win64: 0
+  - first:
+      Any: 
+    second:
+      enabled: 1
+      settings: {}
+  - first:
+      Editor: Editor
+    second:
+      enabled: 1
+      settings:
+        CPU: AnyCPU
+        DefaultValueInitialized: true
+        OS: AnyOS
+  - first:
+      Standalone: Linux64
+    second:
+      enabled: 1
+      settings:
+        CPU: x86_64
+  - first:
+      Standalone: OSXUniversal
+    second:
+      enabled: 1
+      settings:
+        CPU: x86_64
+  - first:
+      Standalone: Win
+    second:
+      enabled: 0
+      settings:
+        CPU: None
+  - first:
+      Standalone: Win64
+    second:
+      enabled: 1
+      settings:
+        CPU: x86_64
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/Plugins/dshowcapture_x64.dll.meta

@@ -0,0 +1,27 @@
+fileFormatVersion: 2
+guid: dd0c04eb3445c3048bfaa964c6bcb16f
+PluginImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  iconMap: {}
+  executionOrder: {}
+  defineConstraints: []
+  isPreloaded: 0
+  isOverridable: 0
+  isExplicitlyReferenced: 0
+  validateReferences: 1
+  platformData:
+  - first:
+      Any: 
+    second:
+      enabled: 1
+      settings: {}
+  - first:
+      Editor: Editor
+    second:
+      enabled: 0
+      settings:
+        DefaultValueInitialized: true
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/Plugins/dshowcapture_x86.dll.meta

@@ -0,0 +1,27 @@
+fileFormatVersion: 2
+guid: a504a0f747083b54c99d6e91731ccecc
+PluginImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  iconMap: {}
+  executionOrder: {}
+  defineConstraints: []
+  isPreloaded: 0
+  isOverridable: 0
+  isExplicitlyReferenced: 0
+  validateReferences: 1
+  platformData:
+  - first:
+      Any: 
+    second:
+      enabled: 1
+      settings: {}
+  - first:
+      Editor: Editor
+    second:
+      enabled: 0
+      settings:
+        DefaultValueInitialized: true
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/Plugins/escapi_x64.dll.meta

@@ -0,0 +1,27 @@
+fileFormatVersion: 2
+guid: dd0c04eb3445c3048bfaa964c6bcb16f
+PluginImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  iconMap: {}
+  executionOrder: {}
+  defineConstraints: []
+  isPreloaded: 0
+  isOverridable: 0
+  isExplicitlyReferenced: 0
+  validateReferences: 1
+  platformData:
+  - first:
+      Any: 
+    second:
+      enabled: 1
+      settings: {}
+  - first:
+      Editor: Editor
+    second:
+      enabled: 0
+      settings:
+        DefaultValueInitialized: true
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/Plugins/escapi_x86.dll.meta

@@ -0,0 +1,27 @@
+fileFormatVersion: 2
+guid: a504a0f747083b54c99d6e91731ccecc
+PluginImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  iconMap: {}
+  executionOrder: {}
+  defineConstraints: []
+  isPreloaded: 0
+  isOverridable: 0
+  isExplicitlyReferenced: 0
+  validateReferences: 1
+  platformData:
+  - first:
+      Any: 
+    second:
+      enabled: 1
+      settings: {}
+  - first:
+      Editor: Editor
+    second:
+      enabled: 0
+      settings:
+        DefaultValueInitialized: true
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/Plugins/libminibmcapture32.dll.meta

@@ -0,0 +1,27 @@
+fileFormatVersion: 2
+guid: 51dc8f0c65df30d4daa188cdfd1dd220
+PluginImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  iconMap: {}
+  executionOrder: {}
+  defineConstraints: []
+  isPreloaded: 0
+  isOverridable: 0
+  isExplicitlyReferenced: 0
+  validateReferences: 1
+  platformData:
+  - first:
+      Any: 
+    second:
+      enabled: 1
+      settings: {}
+  - first:
+      Editor: Editor
+    second:
+      enabled: 0
+      settings:
+        DefaultValueInitialized: true
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/Plugins/libminibmcapture64.dll.meta

@@ -0,0 +1,27 @@
+fileFormatVersion: 2
+guid: fcfc753ca92169e48aa69c1ecc1b0d64
+PluginImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  iconMap: {}
+  executionOrder: {}
+  defineConstraints: []
+  isPreloaded: 0
+  isOverridable: 0
+  isExplicitlyReferenced: 0
+  validateReferences: 1
+  platformData:
+  - first:
+      Any: 
+    second:
+      enabled: 1
+      settings: {}
+  - first:
+      Editor: Editor
+    second:
+      enabled: 0
+      settings:
+        DefaultValueInitialized: true
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/Plugins/thundersvm.dll.meta

@@ -0,0 +1,63 @@
+fileFormatVersion: 2
+guid: e549e6cbc6a547748abc7f1b05bfa7b4
+PluginImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  iconMap: {}
+  executionOrder: {}
+  defineConstraints: []
+  isPreloaded: 0
+  isOverridable: 0
+  isExplicitlyReferenced: 0
+  validateReferences: 1
+  platformData:
+  - first:
+      : Any
+    second:
+      enabled: 0
+      settings:
+        Exclude Editor: 0
+        Exclude Linux64: 0
+        Exclude OSXUniversal: 0
+        Exclude Win: 1
+        Exclude Win64: 0
+  - first:
+      Any: 
+    second:
+      enabled: 1
+      settings: {}
+  - first:
+      Editor: Editor
+    second:
+      enabled: 1
+      settings:
+        CPU: AnyCPU
+        DefaultValueInitialized: true
+        OS: AnyOS
+  - first:
+      Standalone: Linux64
+    second:
+      enabled: 1
+      settings:
+        CPU: x86_64
+  - first:
+      Standalone: OSXUniversal
+    second:
+      enabled: 1
+      settings:
+        CPU: x86_64
+  - first:
+      Standalone: Win
+    second:
+      enabled: 0
+      settings:
+        CPU: None
+  - first:
+      Standalone: Win64
+    second:
+      enabled: 1
+      settings:
+        CPU: x86_64
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

dshowcapture.py

@@ -0,0 +1,341 @@
+import os
+import platform
+import sys
+import numpy as np
+from ctypes import *
+from PIL import Image
+import cv2
+import json
+
+def resolve(name):
+    f = os.path.join(os.path.dirname(__file__), name)
+    return f
+
+lib = None
+bm_lib = None
+bm_options = None
+bm_enabled = False
+
+def set_bm_enabled(v):
+    global bm_enabled
+    bm_enabled = v
+
+def set_options(str):
+    global bm_options
+    bm_options = str
+
+def create_frame_buffer(width, height, factor):
+    buffer = bytearray(width * height * 4 * factor)
+    char_array = c_char * len(buffer)
+    return char_array.from_buffer(buffer)
+
+class DShowCapture():
+    def __init__(self):
+        global lib
+        global bm_lib
+        global bm_options
+        global bm_enabled
+        if lib is None or bm_lib is None:
+            if platform.architecture()[0] == '32bit':
+                dll_path = resolve(os.path.join("dshowcapture", "dshowcapture_x86.dll"))
+                lib = cdll.LoadLibrary(dll_path)
+                if bm_enabled:
+                    dll_path = resolve(os.path.join("dshowcapture", "libminibmcapture32.dll"))
+                    bm_lib = cdll.LoadLibrary(dll_path)
+            else:
+                dll_path = resolve(os.path.join("dshowcapture", "dshowcapture_x64.dll"))
+                lib = cdll.LoadLibrary(dll_path)
+                if bm_enabled:
+                    dll_path = resolve(os.path.join("dshowcapture", "libminibmcapture64.dll"))
+                    bm_lib = cdll.LoadLibrary(dll_path)
+
+            # DirectShow
+            lib.create_capture.restype = c_void_p
+            lib.get_devices.argtypes = [c_void_p]
+            lib.get_device.argtypes = [c_void_p, c_int, c_char_p, c_int]
+            lib.capture_device.argtypes = [c_void_p, c_int, c_int, c_int, c_int]
+            lib.capture_device_by_dcap.argtypes = [c_void_p, c_int, c_int, c_int, c_int, c_longlong]
+            lib.capture_device_default.argtypes = [c_void_p, c_int]
+            lib.get_width.argtypes = [c_void_p]
+            lib.get_height.argtypes = [c_void_p]
+            lib.get_fps.argtypes = [c_void_p]
+            lib.get_flipped.argtypes = [c_void_p]
+            lib.get_colorspace.argtypes = [c_void_p]
+            lib.get_colorspace_internal.argtypes = [c_void_p]
+            lib.capturing.argtypes = [c_void_p]
+            lib.get_frame.argtypes = [c_void_p, c_int, c_char_p, c_int]
+            lib.get_size.argtypes = [c_void_p]
+            lib.stop_capture.argtypes = [c_void_p]
+            lib.get_json_length.argtypes = [c_void_p]
+            lib.get_json.argtypes = [c_void_p, c_char_p, c_int]
+            lib.destroy_capture.argtypes = [c_void_p]
+
+            # Blackmagic
+            if bm_enabled:
+                bm_lib.start_capture_single.argtypes = [c_int, c_int, c_void_p]
+                bm_lib.read_frame_bgra32_blocking.argtypes = [c_char_p, c_int]
+                bm_lib.stop_capture_single.argtypes = []
+                bm_lib.get_json_length.argtypes = []
+                bm_lib.get_json.argtypes = [c_char_p, c_int]
+                if bm_options is not None:
+                    bm_lib.set_options.argtypes = [c_char_p]
+                    bm_lib.set_options(bm_options.encode())
+        self.lib = lib
+        if bm_enabled:
+            self.bm_lib = bm_lib
+        self.cap = lib.create_capture()
+        self.name_buffer = create_string_buffer(255);
+        self.buffer = None
+        self.have_devices = False
+        self.size = None
+        self.real_size = None
+        self.info = None
+        self.type = None
+
+    def __del__(self):
+        if self.buffer is not None:
+            del self.buffer
+        del self.name_buffer
+        self.destroy_capture()
+
+    def get_devices(self):
+        self.have_devices = True
+        return self.lib.get_devices(self.cap)
+
+    def get_device(self, device_number):
+        self.lib.get_device(self.cap, device_number, self.name_buffer, 255)
+        name_str = str(self.name_buffer.value.decode('utf8', 'surrogateescape'))
+        return name_str
+
+    def get_info(self):
+        global bm_enabled
+        # DirectShow
+        json_length = self.lib.get_json_length(self.cap);
+        json_buffer = create_string_buffer(json_length)
+        self.lib.get_json(self.cap, json_buffer, json_length);
+        json_text = str(json_buffer.value.decode('utf8', 'surrogateescape'))
+        self.info = json.loads(json_text)
+        for cam in self.info:
+            cam["type"] = "DirectShow"
+            cam["index"] = cam["id"]
+        
+        # Blackmagic
+        if bm_enabled:
+            json_length = self.bm_lib.get_json_length();
+            json_buffer = create_string_buffer(json_length)
+            self.bm_lib.get_json(json_buffer, json_length);
+            json_text = str(json_buffer.value.decode('utf8', 'surrogateescape'))
+            bm_info = json.loads(json_text)
+            dshow_len = len(self.info)
+            for cam in bm_info:
+                cam["type"] = "Blackmagic"
+                cam["index"] = cam["id"] + dshow_len
+            self.info.extend(bm_info)
+
+        return self.info
+
+    def capture_device(self, cam, width, height, fps):
+        if not self.have_devices:
+            self.get_devices()
+        ret = self.lib.capture_device(self.cap, cam, width, height, fps) == 1
+        if ret:
+            self.type = "DirectShow"
+            self.width = self.get_width()
+            self.height = self.get_height()
+            self.fps = self.get_fps()
+            self.flipped = self.get_flipped()
+            self.colorspace = self.get_colorspace()
+            self.colorspace_internal = self.get_colorspace_internal()
+            self.size = self.width * self.height * 4
+            self.buffer = create_frame_buffer(self.width, self.height, 4)
+        else:
+            self.size = None
+            self.real_size = None
+        return ret;
+
+    def capture_device_by_dcap(self, cam, dcap, width, height, fps):
+        global bm_enabled
+        if not self.have_devices:
+            self.get_devices()
+        if self.info is None:
+            self.get_info()
+        fps = int(10000000 / fps)
+        ret = False
+        if self.info[cam]['type'] == "DirectShow":
+            ret = self.lib.capture_device_by_dcap(self.cap, cam, dcap, width, height, fps) == 1
+        elif bm_enabled and self.info[cam]['type'] == "Blackmagic":
+            ret = self.bm_lib.start_capture_single(self.info[cam]['id'], self.info[cam]['caps'][dcap]['bmModecode'], None) == 1
+        if ret:
+            self.type = self.info[cam]['type']
+            if self.type == "DirectShow":
+                self.width = self.get_width()
+                self.height = self.get_height()
+                self.fps = self.get_fps()
+                self.flipped = self.get_flipped()
+                self.colorspace = self.get_colorspace()
+                self.colorspace_internal = self.get_colorspace_internal()
+                self.size = self.width * self.height * 4 * 4
+                self.buffer = create_frame_buffer(self.width, self.height, 4)
+            elif bm_enabled and self.type == "Blackmagic":
+                self.width = self.info[cam]['caps'][dcap]['minCX']
+                self.height = self.info[cam]['caps'][dcap]['minCY']
+                self.fps = int(10000000 / self.info[cam]['caps'][dcap]['minInterval'])
+                self.flipped = True
+                self.colorspace = 101
+                self.colorspace_internal = self.colorspace
+                self.size = self.width * self.height * 4
+                self.buffer = create_frame_buffer(self.width, self.height, 1)
+                self.real_size = self.size
+        else:
+            self.size = None
+            self.real_size = None
+        return ret;
+
+    def capture_device_default(self, cam):
+        if not self.have_devices:
+            self.get_devices()
+        ret = self.lib.capture_device_default(self.cap, cam) == 1
+        if ret:
+            self.type = "DirectShow"
+            self.width = self.get_width()
+            self.height = self.get_height()
+            self.fps = self.get_fps()
+            self.flipped = self.get_flipped()
+            self.colorspace = self.get_colorspace()
+            self.colorspace_internal = self.get_colorspace_internal()
+            self.size = self.width * self.height * 4 * 4
+            self.buffer = create_frame_buffer(self.width, self.height, 4)
+        else:
+            self.size = None
+            self.real_size = None
+        return ret;
+
+    def get_width(self):
+        return self.lib.get_width(self.cap)
+
+    def get_height(self):
+        return self.lib.get_height(self.cap)
+
+    def get_fps(self):
+        return self.lib.get_fps(self.cap)
+
+    def get_flipped(self):
+        return self.lib.get_flipped(self.cap) != 0
+
+    def get_colorspace(self):
+        colorspace = self.lib.get_colorspace(self.cap)
+        if colorspace == 0:
+            colorspace = self.lib.get_colorspace_internal(self.cap)
+        return colorspace
+
+    def get_colorspace_internal(self):
+        return self.lib.get_colorspace_internal(self.cap)
+
+    def capturing(self):
+        if self.type == "Blackmagic":
+            return 1
+        return self.lib.capturing(self.cap) == 1
+
+    def get_frame(self, timeout):
+        global bm_enabled
+        if self.size is None:
+            return None
+        if self.type == "DirectShow":
+            self.real_size = self.lib.get_frame(self.cap, timeout, self.buffer, self.size)
+        elif bm_enabled and self.type == "Blackmagic":
+            self.bm_lib.read_frame_bgra32_blocking(self.buffer, self.size)
+        else:
+            return None
+        img = np.frombuffer(self.buffer, dtype=np.uint8)[0:self.real_size]
+        if self.colorspace in [100, 101]:
+            if self.real_size == self.height * self.width * 4:
+                img = cv2.cvtColor(img.reshape((self.height,self.width,4)), cv2.COLOR_BGRA2BGR)
+            elif self.real_size == self.height * self.width * 3:
+                img = img.reshape((self.height,self.width,3))
+            else:
+                return None
+            if not self.flipped:
+                img = cv2.flip(img, 0)
+            return img
+        elif self.colorspace == 200 and self.real_size == (3 * self.height // 2) * self.width:
+            img = cv2.cvtColor(img.reshape((3 * self.height // 2,self.width,1)), cv2.COLOR_YUV2BGR_I420)
+        elif self.colorspace == 201 and self.real_size == (3 * self.height // 2) * self.width:
+            img = cv2.cvtColor(img.reshape((3 * self.height // 2,self.width,1)), cv2.COLOR_YUV2BGR_NV12)
+        elif self.colorspace == 202 and self.real_size == (3 * self.height // 2) * self.width:
+            img = cv2.cvtColor(img.reshape((3 * self.height // 2,self.width,1)), cv2.COLOR_YUV2BGR_YV12)
+        elif self.colorspace == 203 and self.real_size == self.height * self.width:
+            img = cv2.cvtColor(img.reshape((self.height,self.width,1)), cv2.COLOR_GRAY2BGR)
+        elif self.colorspace == 300 and self.real_size == self.height * self.width * 2:
+            img = cv2.cvtColor(img.reshape((self.height,self.width,2)), cv2.COLOR_YUV2BGR_YVYU)
+        elif self.colorspace == 301 and self.real_size == self.height * self.width * 2:
+            img = cv2.cvtColor(img.reshape((self.height,self.width,2)), cv2.COLOR_YUV2BGR_YUY2)
+        elif self.colorspace == 302 and self.real_size == self.height * self.width * 2:
+            img = cv2.cvtColor(img.reshape((self.height,self.width,2)), cv2.COLOR_YUV2BGR_UYVY)
+        elif self.colorspace == 303:
+            return None
+        elif self.colorspace == 400 and self.real_size > 4:
+            img = cv2.imdecode(img, cv2.IMREAD_COLOR);
+        else:
+            return None
+        if self.flipped:
+            img = cv2.flip(img, 0)
+        return img
+
+    def stop_capture(self):
+        global bm_enabled
+        self.size = None
+        self.real_size = None
+        if self.type == "DirectShow":
+            return self.lib.stop_capture(self.cap)
+        elif bm_enabled and self.type == "Blackmagic":
+            return self.bm_lib.stop_capture_single() != 0
+        return True
+
+    def destroy_capture(self):
+        if self.cap is None:
+            return 0
+        self.stop_capture()
+        ret = self.lib.destroy_capture(self.cap)
+        self.cap = None
+        self.size = None
+        self.real_size = None
+        return ret
+
+if __name__ == "__main__":
+    cam = 0
+    width = 1280
+    height = 720
+    fps = 30
+    if len(sys.argv) > 1:
+        cam = int(sys.argv[1])
+    if len(sys.argv) > 2:
+        width = int(sys.argv[2])
+    if len(sys.argv) > 3:
+        height = int(sys.argv[3])
+    if len(sys.argv) > 4:
+        fps = int(sys.argv[4])
+
+    cap = DShowCapture()
+    devices = cap.get_devices()
+    print("Devices: ", devices)
+    for i in range(devices):
+        print(f"{i} {cap.get_device(i)}")
+
+    print(f"Capturing: {cap.capture_device(cam, width, height, fps)}")
+    width = cap.get_width()
+    height = cap.get_height()
+    flipped = cap.get_flipped()
+    print(f"Width: {width} Height: {height} FPS: {cap.get_fps()} Flipped: {flipped} Colorspace: {cap.get_colorspace()} Internal: {cap.get_colorspace_internal()}")
+    
+    while True:
+        img = cap.get_frame(1000)
+        if img is not None:
+            cv2.imshow("DShowCapture", img)
+            if cv2.waitKey(1) & 0xFF == ord('q'):
+                sys.exit(0)
+        else:
+            print("Lost frame")
+
+    cap.stop_capture()
+    cap.destroy_capture()
+    del cap

facetracker.py

@@ -1,50 +1,127 @@
+import copy
 import os
 import sys
 import argparse
+import traceback
+import gc
 
 parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 parser.add_argument("-i", "--ip", help="Set IP address for sending tracking data", default="127.0.0.1")
 parser.add_argument("-p", "--port", type=int, help="Set port for sending tracking data", default=11573)
 if os.name == 'nt':
-    parser.add_argument("-l", "--list-cameras", type=int, help="Set this to 1 to list the available cameras and quit, set this to a number higher than 1 to output only the names", default=0)
+    parser.add_argument("-l", "--list-cameras", type=int, help="Set this to 1 to list the available cameras and quit, set this to 2 or higher to output only the names", default=0)
+    parser.add_argument("-a", "--list-dcaps", type=int, help="Set this to -1 to list all cameras and their available capabilities, set this to a camera id to list that camera's capabilities", default=None)
     parser.add_argument("-W", "--width", type=int, help="Set camera and raw RGB width", default=640)
     parser.add_argument("-H", "--height", type=int, help="Set camera and raw RGB height", default=360)
-    parser.add_argument("-F", "--fps", type=int, help="Set camera frames per second", default=24)
+    parser.add_argument("-D", "--dcap", type=int, help="Set which device capability line to use or -1 to use the default camera settings (FPS still need to be set separately)", default=None)
+    parser.add_argument("-B", "--blackmagic", type=int, help="When set to 1, special support for Blackmagic devices is enabled", default=0)
 else:
     parser.add_argument("-W", "--width", type=int, help="Set raw RGB width", default=640)
     parser.add_argument("-H", "--height", type=int, help="Set raw RGB height", default=360)
-parser.add_argument("-c", "--capture", help="Set camera ID (0, 1...) or video file", default=0)
+if sys.platform == 'linux':
+    parser.add_argument("--dformat", type=str, help="Set which device format to use (MJPG, YUYV, RGB3, ...)", default=None)
+parser.add_argument("-F", "--fps", type=int, help="Set camera frames per second", default=24)
+parser.add_argument("-c", "--capture", help="Set camera ID (0, 1...) or video file", default="0")
+parser.add_argument("-M", "--mirror-input", action="store_true", help="Process a mirror image of the input video")
 parser.add_argument("-m", "--max-threads", type=int, help="Set the maximum number of threads", default=1)
-parser.add_argument("-t", "--threshold", type=float, help="Set minimum confidence threshold for face detection", default=0.4)
+parser.add_argument("-t", "--threshold", type=float, help="Set minimum confidence threshold for face tracking", default=None)
-parser.add_argument("-v", "--visualize", type=int, help="Set this to 1 to visualize the tracking or to 2 to add numbers to the point display", default=0)
+parser.add_argument("-d", "--detection-threshold", type=float, help="Set minimum confidence threshold for face detection", default=0.6)
+parser.add_argument("-v", "--visualize", type=int, help="Set this to 1 to visualize the tracking, to 2 to also show face ids, to 3 to add confidence values or to 4 to add numbers to the point display", default=0)
 parser.add_argument("-P", "--pnp-points", type=int, help="Set this to 1 to add the 3D fitting points to the visualization", default=0)
 parser.add_argument("-s", "--silent", type=int, help="Set this to 1 to prevent text output on the console", default=0)
 parser.add_argument("--faces", type=int, help="Set the maximum number of faces (slow)", default=1)
+parser.add_argument("--scan-retinaface", type=int, help="When set to 1, scanning for additional faces will be performed using RetinaFace in a background thread, otherwise a simpler, faster face detection mechanism is used. When the maximum number of faces is 1, this option does nothing.", default=0)
 parser.add_argument("--scan-every", type=int, help="Set after how many frames a scan for new faces should run", default=3)
 parser.add_argument("--discard-after", type=int, help="Set the how long the tracker should keep looking for lost faces", default=10)
+parser.add_argument("--max-feature-updates", type=int, help="This is the number of seconds after which feature min/max/medium values will no longer be updated once a face has been detected.", default=900)
+parser.add_argument("--no-3d-adapt", type=int, help="When set to 1, the 3D face model will not be adapted to increase the fit", default=1)
+parser.add_argument("--try-hard", type=int, help="When set to 1, the tracker will try harder to find a face", default=0)
 parser.add_argument("--video-out", help="Set this to the filename of an AVI file to save the tracking visualization as a video", default=None)
+parser.add_argument("--video-scale", type=int, help="This is a resolution scale factor applied to the saved AVI file", default=1, choices=[1,2,3,4])
+parser.add_argument("--video-fps", type=float, help="This sets the frame rate of the output AVI file", default=24)
 parser.add_argument("--raw-rgb", type=int, help="When this is set, raw RGB frames of the size given with \"-W\" and \"-H\" are read from standard input instead of reading a video", default=0)
 parser.add_argument("--log-data", help="You can set a filename to which tracking data will be logged here", default="")
-parser.add_argument("--model", type=int, help="This can be used to select between the two models with 0 being the original model that might be overall more accurate, 1 being a model that might be able to fit eye and eyebrow shapes more accurately and 2 being a much faster, but lower accuracy model", default=1, choices=[0, 1, 2])
+parser.add_argument("--log-output", help="You can set a filename to console output will be logged here", default="")
+parser.add_argument("--model", type=int, help="This can be used to select the tracking model. Higher numbers are models with better tracking quality, but slower speed, except for model 4, which is wink optimized. Models 1 and 0 tend to be too rigid for expression and blink detection. Model -2 is roughly equivalent to model 1, but faster. Model -3 is between models 0 and -1.", default=3, choices=[-3, -2, -1, 0, 1, 2, 3, 4])
 parser.add_argument("--model-dir", help="This can be used to specify the path to the directory containing the .onnx model files", default=None)
-parser.add_argument("--high-quality-3d", type=int, help="When set to 1, more nose points are used when estimating the face pose", default=1)
+parser.add_argument("--gaze-tracking", type=int, help="When set to 1, gaze tracking is enabled, which makes things slightly slower", default=1)
-parser.add_argument("--gaze-tracking", type=int, help="When set to 1, experimental blink detection and gaze tracking are enabled, which makes things slightly slower", default=1)
+parser.add_argument("--face-id-offset", type=int, help="When set, this offset is added to all face ids, which can be useful for mixing tracking data from multiple network sources", default=0)
+parser.add_argument("--repeat-video", type=int, help="When set to 1 and a video file was specified with -c, the tracker will loop the video until interrupted", default=0)
+parser.add_argument("--dump-points", type=str, help="When set to a filename, the current face 3D points are made symmetric and dumped to the given file when quitting the visualization with the \"q\" key", default="")
+parser.add_argument("--benchmark", type=int, help="When set to 1, the different tracking models are benchmarked, starting with the best and ending with the fastest and with gaze tracking disabled for models with negative IDs", default=0)
+if os.name == 'nt':
+    parser.add_argument("--use-dshowcapture", type=int, help="When set to 1, libdshowcapture will be used for video input instead of OpenCV", default=1)
+    parser.add_argument("--blackmagic-options", type=str, help="When set, this additional option string is passed to the blackmagic capture library", default=None)
+    parser.add_argument("--priority", type=int, help="When set, the process priority will be changed", default=None, choices=[0, 1, 2, 3, 4, 5])
 args = parser.parse_args()
 
-os.environ["OMP_NUM_THREADS"] = "1"
+os.environ["OMP_NUM_THREADS"] = str(args.max_threads)
 
-if os.name == 'nt' and args.list_cameras > 0:
+class OutputLog(object):
-    import escapi
+    def __init__(self, fh, output):
-    escapi.init()
+        self.fh = fh
-    camera_count = escapi.count_capture_devices()
+        self.output = output
-    if args.list_cameras == 1:
+    def write(self, buf):
-        print("Available cameras:")
+        if self.fh is not None:
-    for i in range(camera_count):
+            self.fh.write(buf)
-        camera_name = escapi.device_name(i).decode()
+        self.output.write(buf)
+        self.flush()
+    def flush(self):
+        if self.fh is not None:
+            self.fh.flush()
+        self.output.flush()
+output_logfile = None
+if args.log_output != "":
+    output_logfile = open(args.log_output, "w")
+sys.stdout = OutputLog(output_logfile, sys.stdout)
+sys.stderr = OutputLog(output_logfile, sys.stderr)
+
+if os.name == 'nt':
+    import dshowcapture
+    if args.blackmagic == 1:
+        dshowcapture.set_bm_enabled(True)
+    if args.blackmagic_options is not None:
+        dshowcapture.set_options(args.blackmagic_options)
+    if args.priority is not None:
+        import psutil
+        classes = [psutil.IDLE_PRIORITY_CLASS, psutil.BELOW_NORMAL_PRIORITY_CLASS, psutil.NORMAL_PRIORITY_CLASS, psutil.ABOVE_NORMAL_PRIORITY_CLASS, psutil.HIGH_PRIORITY_CLASS, psutil.REALTIME_PRIORITY_CLASS]
+        p = psutil.Process(os.getpid())
+        p.nice(classes[args.priority])
+
+if os.name == 'nt' and (args.list_cameras > 0 or args.list_dcaps is not None):
+    cap = dshowcapture.DShowCapture()
+    info = cap.get_info()
+    unit = 10000000.;
+    if args.list_dcaps is not None:
+        formats = {0: "Any", 1: "Unknown", 100: "ARGB", 101: "XRGB", 200: "I420", 201: "NV12", 202: "YV12", 203: "Y800", 300: "YVYU", 301: "YUY2", 302: "UYVY", 303: "HDYC (Unsupported)", 400: "MJPEG", 401: "H264" }
+        for cam in info:
+            if args.list_dcaps == -1:
+                type = ""
+                if cam['type'] == "Blackmagic":
+                    type = "Blackmagic: "
+                print(f"{cam['index']}: {type}{cam['name']}")
+            if args.list_dcaps != -1 and args.list_dcaps != cam['index']:
+                continue
+            for caps in cam['caps']:
+                format = caps['format']
+                if caps['format'] in formats:
+                    format = formats[caps['format']]
+                if caps['minCX'] == caps['maxCX'] and caps['minCY'] == caps['maxCY']:
+                    print(f"    {caps['id']}: Resolution: {caps['minCX']}x{caps['minCY']} FPS: {unit/caps['maxInterval']:.3f}-{unit/caps['minInterval']:.3f} Format: {format}")
+                else:
+                    print(f"    {caps['id']}: Resolution: {caps['minCX']}x{caps['minCY']}-{caps['maxCX']}x{caps['maxCY']} FPS: {unit/caps['maxInterval']:.3f}-{unit/caps['minInterval']:.3f} Format: {format}")
+    else:
         if args.list_cameras == 1:
-            print(f"{i}: {camera_name}")
+            print("Available cameras:")
-        else:
+        for cam in info:
-            print(camera_name)
+            type = ""
+            if cam['type'] == "Blackmagic":
+                type = "Blackmagic: "
+            if args.list_cameras == 1:
+                print(f"{cam['index']}: {type}{cam['name']}")
+            else:
+                print(f"{type}{cam['name']}")
+    cap.destroy_capture()
     sys.exit(0)
 
 import numpy as np
@@ -52,8 +129,25 @@ import time
 import cv2
 import socket
 import struct
-from input_reader import InputReader, list_cameras
+import json
-from tracker import Tracker
+from input_reader import InputReader, VideoReader, DShowCaptureReader, try_int
+from tracker import Tracker, get_model_base_path
+
+if args.benchmark > 0:
+    model_base_path = get_model_base_path(args.model_dir)
+    im = cv2.imread(os.path.join(model_base_path, "benchmark.bin"), cv2.IMREAD_COLOR)
+    results = []
+    for model_type in [3, 2, 1, 0, -1, -2, -3]:
+        tracker = Tracker(224, 224, threshold=0.1, max_threads=args.max_threads, max_faces=1, discard_after=0, scan_every=0, silent=True, model_type=model_type, model_dir=args.model_dir, no_gaze=(model_type == -1), detection_threshold=0.1, use_retinaface=0, max_feature_updates=900, static_model=True if args.no_3d_adapt == 1 else False)
+        tracker.detected = 1
+        tracker.faces = [(0, 0, 224, 224)]
+        total = 0.0
+        for i in range(100):
+            start = time.perf_counter()
+            r = tracker.predict(im)
+            total += time.perf_counter() - start
+        print(1. / (total / 100.))
+    sys.exit(0)
 
 target_ip = args.ip
 target_port = args.port
@@ -61,10 +155,21 @@ target_port = args.port
 if args.faces >= 40:
     print("Transmission of tracking data over network is not supported with 40 or more faces.")
 
-fps = 0
+fps = args.fps
+dcap = None
+use_dshowcapture_flag = False
 if os.name == 'nt':
-    fps = args.fps
+    dcap = args.dcap
-input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps)
+    use_dshowcapture_flag = True if args.use_dshowcapture == 1 else False
+    input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps, use_dshowcapture=use_dshowcapture_flag, dcap=dcap)
+    if args.dcap == -1 and type(input_reader) == DShowCaptureReader:
+        fps = min(fps, input_reader.device.get_fps())
+elif sys.platform == 'linux' and args.dformat:
+    input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps, dcap=args.dformat)
+else:
+    input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps)
+if type(input_reader.reader) == VideoReader:
+    fps = 0
 
 log = None
 out = None
@@ -73,30 +178,72 @@ height = 0
 width = 0
 tracker = None
 sock = None
+total_tracking_time = 0.0
 tracking_time = 0.0
 tracking_frames = 0
 frame_count = 0
 
+features = ["eye_l", "eye_r", "eyebrow_steepness_l", "eyebrow_updown_l", "eyebrow_quirk_l", "eyebrow_steepness_r", "eyebrow_updown_r", "eyebrow_quirk_r", "mouth_corner_updown_l", "mouth_corner_inout_l", "mouth_corner_updown_r", "mouth_corner_inout_r", "mouth_open", "mouth_wide"]
+
 if args.log_data != "":
     log = open(args.log_data, "w")
-    log.write("Frame,Time,Face,AverageConfidence,Success3D,PnPError,RotationQuat.X,RotationQuat.Y,RotationQuat.Z,RotationQuat.W,RVec.X,RVec.Y,RVec.Z,TVec.X,TVec.Y,TVec.Z")
+    log.write("Frame,Time,Width,Height,FPS,Face,FaceID,RightOpen,LeftOpen,AverageConfidence,Success3D,PnPError,RotationQuat.X,RotationQuat.Y,RotationQuat.Z,RotationQuat.W,Euler.X,Euler.Y,Euler.Z,RVec.X,RVec.Y,RVec.Z,TVec.X,TVec.Y,TVec.Z")
-    for i in range(66):
+    for i in range(68):
         log.write(f",Landmark[{i}].X,Landmark[{i}].Y,Landmark[{i}].Confidence")
-    for i in range(66):
+    for i in range(70):
         log.write(f",Point3D[{i}].X,Point3D[{i}].Y,Point3D[{i}].Z")
+    for feature in features:
+        log.write(f",{feature}")
     log.write("\r\n")
     log.flush()
 
+is_camera = args.capture == str(try_int(args.capture))
+
 try:
-    while input_reader.is_open():
+    attempt = 0
+    frame_time = time.perf_counter()
+    target_duration = 0
+    if fps > 0:
+        target_duration = 1. / float(fps)
+    repeat = args.repeat_video != 0 and type(input_reader.reader) == VideoReader
+    need_reinit = 0
+    failures = 0
+    source_name = input_reader.name
+    while repeat or input_reader.is_open():
+        if not input_reader.is_open() or need_reinit == 1:
+            input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps, use_dshowcapture=use_dshowcapture_flag, dcap=dcap)
+            if input_reader.name != source_name:
+                print(f"Failed to reinitialize camera and got {input_reader.name} instead of {source_name}.")
+                sys.exit(1)
+            need_reinit = 2
+            time.sleep(0.02)
+            continue
         if not input_reader.is_ready():
-            time.sleep(0.001)
+            time.sleep(0.02)
             continue
 
         ret, frame = input_reader.read()
+        if ret and args.mirror_input:
+            frame = cv2.flip(frame, 1)
         if not ret:
-            break
+            if repeat:
+                if need_reinit == 0:
+                    need_reinit = 1
+                continue
+            elif is_camera:
+                attempt += 1
+                if attempt > 30:
+                    break
+                else:
+                    time.sleep(0.02)
+                    if attempt == 3:
+                        need_reinit = 1
+                    continue
+            else:
+                break;
 
+        attempt = 0
+        need_reinit = 0
         frame_count += 1
         now = time.time()
 
@@ -104,108 +251,210 @@ try:
             first = False
             height, width, channels = frame.shape
             sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-            tracker = Tracker(width, height, threshold=args.threshold, max_threads=args.max_threads, max_faces=args.faces, discard_after=args.discard_after, scan_every=args.scan_every, silent=False if args.silent == 0 else True, model_type=args.model, pnp_quality=args.high_quality_3d, model_dir=args.model_dir, no_gaze=False if args.gaze_tracking != 0 else True)
+            tracker = Tracker(width, height, threshold=args.threshold, max_threads=args.max_threads, max_faces=args.faces, discard_after=args.discard_after, scan_every=args.scan_every, silent=False if args.silent == 0 else True, model_type=args.model, model_dir=args.model_dir, no_gaze=False if args.gaze_tracking != 0 and args.model != -1 else True, detection_threshold=args.detection_threshold, use_retinaface=args.scan_retinaface, max_feature_updates=args.max_feature_updates, static_model=True if args.no_3d_adapt == 1 else False, try_hard=args.try_hard == 1)
-            if not args.video_out is None:
+            if args.video_out is not None:
-                out = cv2.VideoWriter(args.video_out, cv2.VideoWriter_fourcc('M','J','P','G'), 24, (width,height))
+                out = cv2.VideoWriter(args.video_out, cv2.VideoWriter_fourcc('F','F','V','1'), args.video_fps, (width * args.video_scale, height * args.video_scale))
 
-        inference_start = time.time()
+        try:
-        faces = tracker.predict(frame)
+            inference_start = time.perf_counter()
-        if len(faces) > 0:
+            faces = tracker.predict(frame)
-            tracking_time += (time.time() - inference_start) / len(faces)
+            if len(faces) > 0:
-            tracking_frames += 1
+                inference_time = (time.perf_counter() - inference_start)
-        packet = bytearray()
+                total_tracking_time += inference_time
-        detected = False
+                tracking_time += inference_time / len(faces)
-        for face_num, (conf, lms, success_3d, pnp_error, quaternion, euler, rotation, translation, pts_3d, (right_open, left_open)) in enumerate(faces): # TODO: eye_state
+                tracking_frames += 1
-            if args.silent == 0:
+            packet = bytearray()
-                print(f"Confidence: {conf:.4f} / 3D fitting error: {pnp_error:.4f}")
+            detected = False
-            detected = True
+            for face_num, f in enumerate(faces):
-            if not success_3d:
+                f = copy.copy(f)
-                pts_3d = np.zeros((70, 3), np.float32)
+                f.id += args.face_id_offset
-            packet.extend(bytearray(struct.pack("d", now)))
+                if f.eye_blink is None:
-            packet.extend(bytearray(struct.pack("f", right_open)))
+                    f.eye_blink = [1, 1]
-            packet.extend(bytearray(struct.pack("f", left_open)))
+                right_state = "O" if f.eye_blink[0] > 0.30 else "-"
-            packet.extend(bytearray(struct.pack("B", 1 if success_3d else 0)))
+                left_state = "O" if f.eye_blink[1] > 0.30 else "-"
-            packet.extend(bytearray(struct.pack("f", pnp_error)))
+                if args.silent == 0:
-            packet.extend(bytearray(struct.pack("f", quaternion[0])))
+                    print(f"Confidence[{f.id}]: {f.conf:.4f} / 3D fitting error: {f.pnp_error:.4f} / Eyes: {left_state}, {right_state}")
-            packet.extend(bytearray(struct.pack("f", quaternion[1])))
+                detected = True
-            packet.extend(bytearray(struct.pack("f", quaternion[2])))
+                if not f.success:
-            packet.extend(bytearray(struct.pack("f", quaternion[3])))
+                    pts_3d = np.zeros((70, 3), np.float32)
-            packet.extend(bytearray(struct.pack("f", euler[0])))
+                packet.extend(bytearray(struct.pack("d", now)))
-            packet.extend(bytearray(struct.pack("f", euler[1])))
+                packet.extend(bytearray(struct.pack("i", f.id)))
-            packet.extend(bytearray(struct.pack("f", euler[2])))
+                packet.extend(bytearray(struct.pack("f", width)))
-            packet.extend(bytearray(struct.pack("f", translation[0][0])))
+                packet.extend(bytearray(struct.pack("f", height)))
-            packet.extend(bytearray(struct.pack("f", translation[0][1])))
+                packet.extend(bytearray(struct.pack("f", f.eye_blink[0])))
-            packet.extend(bytearray(struct.pack("f", translation[0][2])))
+                packet.extend(bytearray(struct.pack("f", f.eye_blink[1])))
-            if not log is None:
+                packet.extend(bytearray(struct.pack("B", 1 if f.success else 0)))
-                log.write(f"{frame_count},{now},{face_num},{conf},{success_3d},{pnp_error},{quaternion[0]},{quaternion[1]},{quaternion[2]},{quaternion[3]},{rotation[0][0]},{rotation[0][1]},{rotation[0][2]},{translation[0][0]},{translation[0][1]},{translation[0][2]}")
+                packet.extend(bytearray(struct.pack("f", f.pnp_error)))
-            for (x,y,c) in lms:
+                packet.extend(bytearray(struct.pack("f", f.quaternion[0])))
-                packet.extend(bytearray(struct.pack("f", c)))
+                packet.extend(bytearray(struct.pack("f", f.quaternion[1])))
-            for pt_num, (x,y,c) in enumerate(lms):
+                packet.extend(bytearray(struct.pack("f", f.quaternion[2])))
-                packet.extend(bytearray(struct.pack("f", y)))
+                packet.extend(bytearray(struct.pack("f", f.quaternion[3])))
-                packet.extend(bytearray(struct.pack("f", x)))
+                packet.extend(bytearray(struct.pack("f", f.euler[0])))
-                if not log is None:
+                packet.extend(bytearray(struct.pack("f", f.euler[1])))
-                    log.write(f",{y},{x},{c}")
+                packet.extend(bytearray(struct.pack("f", f.euler[2])))
-                if pt_num == 66:# and right_open < 0.5:
+                packet.extend(bytearray(struct.pack("f", f.translation[0])))
-                    continue
+                packet.extend(bytearray(struct.pack("f", f.translation[1])))
-                if pt_num == 67:# and left_open < 0.5:
+                packet.extend(bytearray(struct.pack("f", f.translation[2])))
-                    continue
+                if log is not None:
-                if args.visualize != 0 or not out is None:
+                    log.write(f"{frame_count},{now},{width},{height},{fps},{face_num},{f.id},{f.eye_blink[0]},{f.eye_blink[1]},{f.conf},{f.success},{f.pnp_error},{f.quaternion[0]},{f.quaternion[1]},{f.quaternion[2]},{f.quaternion[3]},{f.euler[0]},{f.euler[1]},{f.euler[2]},{f.rotation[0]},{f.rotation[1]},{f.rotation[2]},{f.translation[0]},{f.translation[1]},{f.translation[2]}")
-                    if args.visualize > 1:
+                for (x,y,c) in f.lms:
-                        frame = cv2.putText(frame, str(pt_num), (int(y), int(x)), cv2.FONT_HERSHEY_SIMPLEX, 0.25, (255,255,0))
+                    packet.extend(bytearray(struct.pack("f", c)))
-                    if not (x < 0 or y < 0 or x >= height or y >= width):
+                if args.visualize > 1:
-                        frame[int(x), int(y)] = (0, 0, 255)
+                    frame = cv2.putText(frame, str(f.id), (int(f.bbox[0]), int(f.bbox[1])), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (255,0,255))
-                    x += 1
+                if args.visualize > 2:
-                    if not (x < 0 or y < 0 or x >= height or y >= width):
+                    frame = cv2.putText(frame, f"{f.conf:.4f}", (int(f.bbox[0] + 18), int(f.bbox[1] - 6)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,0,255))
-                        frame[int(x), int(y)] = (0, 0, 255)
+                for pt_num, (x,y,c) in enumerate(f.lms):
-                    y += 1
+                    packet.extend(bytearray(struct.pack("f", y)))
-                    if not (x < 0 or y < 0 or x >= height or y >= width):
+                    packet.extend(bytearray(struct.pack("f", x)))
-                        frame[int(x), int(y)] = (0, 0, 255)
+                    if log is not None:
-                    x -= 1
+                        log.write(f",{y},{x},{c}")
-                    if not (x < 0 or y < 0 or x >= height or y >= width):
+                    if pt_num == 66 and (f.eye_blink[0] < 0.30 or c < 0.20):
-                        frame[int(x), int(y)] = (0, 0, 255)
+                        continue
-            if args.pnp_points != 0 and (args.visualize != 0 or not out is None):
+                    if pt_num == 67 and (f.eye_blink[1] < 0.30 or c < 0.20):
-                projected = cv2.projectPoints(tracker.contour, rotation, translation, tracker.camera, tracker.dist_coeffs)
+                        continue
-                for [(x,y)] in projected[0]:
+                    x = int(x + 0.5)
-                    if not (x < 0 or y < 0 or x >= height or y >= width):
+                    y = int(y + 0.5)
-                        frame[int(x), int(y)] = (0, 255, 255)
+                    if args.visualize != 0 or out is not None:
-                    x += 1
+                        if args.visualize > 3:
-                    if not (x < 0 or y < 0 or x >= height or y >= width):
+                            frame = cv2.putText(frame, str(pt_num), (int(y), int(x)), cv2.FONT_HERSHEY_SIMPLEX, 0.25, (255,255,0))
-                        frame[int(x), int(y)] = (0, 255, 255)
+                        color = (0, 255, 0)
-                    y += 1
+                        if pt_num >= 66:
-                    if not (x < 0 or y < 0 or x >= height or y >= width):
+                            color = (255, 255, 0)
-                        frame[int(x), int(y)] = (0, 255, 255)
+                        if not (x < 0 or y < 0 or x >= height or y >= width):
-                    x -= 1
+                            cv2.circle(frame, (y, x), 1, color, -1)
-                    if not (x < 0 or y < 0 or x >= height or y >= width):
+                if args.pnp_points != 0 and (args.visualize != 0 or out is not None) and f.rotation is not None:
-                        frame[int(x), int(y)] = (0, 255, 255)
+                    if args.pnp_points > 1:
-            for (x,y,z) in pts_3d:
+                        projected = cv2.projectPoints(f.face_3d[0:66], f.rotation, f.translation, tracker.camera, tracker.dist_coeffs)
-                packet.extend(bytearray(struct.pack("f", x)))
+                    else:
-                packet.extend(bytearray(struct.pack("f", -y)))
+                        projected = cv2.projectPoints(f.contour, f.rotation, f.translation, tracker.camera, tracker.dist_coeffs)
-                packet.extend(bytearray(struct.pack("f", -z)))
+                    for [(x,y)] in projected[0]:
-                if not log is None:
+                        x = int(x + 0.5)
-                    log.write(f",{x},{-y},{-z}")
+                        y = int(y + 0.5)
-            if not log is None:
+                        if not (x < 0 or y < 0 or x >= height or y >= width):
-                log.write("\r\n")
+                            frame[int(x), int(y)] = (0, 255, 255)
-                log.flush()
+                        x += 1
+                        if not (x < 0 or y < 0 or x >= height or y >= width):
+                            frame[int(x), int(y)] = (0, 255, 255)
+                        y += 1
+                        if not (x < 0 or y < 0 or x >= height or y >= width):
+                            frame[int(x), int(y)] = (0, 255, 255)
+                        x -= 1
+                        if not (x < 0 or y < 0 or x >= height or y >= width):
+                            frame[int(x), int(y)] = (0, 255, 255)
+                for (x,y,z) in f.pts_3d:
+                    packet.extend(bytearray(struct.pack("f", x)))
+                    packet.extend(bytearray(struct.pack("f", -y)))
+                    packet.extend(bytearray(struct.pack("f", -z)))
+                    if log is not None:
+                        log.write(f",{x},{-y},{-z}")
+                if f.current_features is None:
+                    f.current_features = {}
+                for feature in features:
+                    if not feature in f.current_features:
+                        f.current_features[feature] = 0
+                    packet.extend(bytearray(struct.pack("f", f.current_features[feature])))
+                    if log is not None:
+                        log.write(f",{f.current_features[feature]}")
+                if log is not None:
+                    log.write("\r\n")
+                    log.flush()
 
-        if detected and len(faces) < 40:
+            if detected and len(faces) < 40:
-            sock.sendto(packet, (target_ip, target_port))
+                sock.sendto(packet, (target_ip, target_port))
 
-        if not out is None:
+            if out is not None:
-            out.write(frame)
+                video_frame = frame
+                if args.video_scale != 1:
+                    video_frame = cv2.resize(frame, (width * args.video_scale, height * args.video_scale), interpolation=cv2.INTER_NEAREST)
+                out.write(video_frame)
+                if args.video_scale != 1:
+                    del video_frame
 
-        if args.visualize != 0:
+            if args.visualize != 0:
-            cv2.imshow('Frame', frame)
+                cv2.imshow('OpenSeeFace Visualization', frame)
-            if cv2.waitKey(1) & 0xFF == ord('q'):
+                if cv2.waitKey(1) & 0xFF == ord('q'):
+                    if args.dump_points != "" and faces is not None and len(faces) > 0:
+                        np.set_printoptions(threshold=sys.maxsize, precision=15)
+                        pairs = [
+                            (0, 16),
+                            (1, 15),
+                            (2, 14),
+                            (3, 13),
+                            (4, 12),
+                            (5, 11),
+                            (6, 10),
+                            (7, 9),
+                            (17, 26),
+                            (18, 25),
+                            (19, 24),
+                            (20, 23),
+                            (21, 22),
+                            (31, 35),
+                            (32, 34),
+                            (36, 45),
+                            (37, 44),
+                            (38, 43),
+                            (39, 42),
+                            (40, 47),
+                            (41, 46),
+                            (48, 52),
+                            (49, 51),
+                            (56, 54),
+                            (57, 53),
+                            (58, 62),
+                            (59, 61),
+                            (65, 63)
+                        ]
+                        points = copy.copy(faces[0].face_3d)
+                        for a, b in pairs:
+                            x = (points[a, 0] - points[b, 0]) / 2.0
+                            y = (points[a, 1] + points[b, 1]) / 2.0
+                            z = (points[a, 2] + points[b, 2]) / 2.0
+                            points[a, 0] = x
+                            points[b, 0] = -x
+                            points[[a, b], 1] = y
+                            points[[a, b], 2] = z
+                        points[[8, 27, 28, 29, 33, 50, 55, 60, 64], 0] = 0.0
+                        points[30, :] = 0.0
+                        with open(args.dump_points, "w") as fh:
+                            fh.write(repr(points))
+                    break
+            failures = 0
+        except Exception as e:
+            if e.__class__ == KeyboardInterrupt:
+                if args.silent == 0:
+                    print("Quitting")
                 break
+            traceback.print_exc()
+            failures += 1
+            if failures > 30:
+                break
+
+        collected = False
+        del frame
+
+        duration = time.perf_counter() - frame_time
+        while duration < target_duration:
+            if not collected:
+                gc.collect()
+                collected = True
+            duration = time.perf_counter() - frame_time
+            sleep_time = target_duration - duration
+            if sleep_time > 0:
+                time.sleep(sleep_time)
+            duration = time.perf_counter() - frame_time
+        frame_time = time.perf_counter()
 except KeyboardInterrupt:
     if args.silent == 0:
         print("Quitting")
 
 input_reader.close()
-if not out is None:
+if out is not None:
     out.release()
-cv2.destroyAllWindows()
+if args.visualize != 0:
+    cv2.destroyAllWindows()
 
 if args.silent == 0 and tracking_frames > 0:
-    tracking_time = 1000 * tracking_time / tracking_frames
+    average_tracking_time = 1000 * tracking_time / tracking_frames
-    print(f"Average tracking time per detected face: {tracking_time:.2f} ms")
+    print(f"Average tracking time per detected face: {average_tracking_time:.2f} ms")
+    print(f"Tracking time: {total_tracking_time:.3f} s\nFrames: {tracking_frames}")

facetracker.spec

@@ -0,0 +1,53 @@
+# -*- mode: python ; coding: utf-8 -*-
+
+block_cipher = None
+
+
+a = Analysis(['facetracker.py'],
+             pathex=['C:\\OpenSeeFaceBuild'],
+             binaries=[('dshowcapture/dshowcapture_x86.dll', '.'), ('dshowcapture/dshowcapture_x64.dll', '.'), ('dshowcapture/libminibmcapture32.dll', '.'), ('dshowcapture/libminibmcapture64.dll', '.'), ('escapi/escapi_x86.dll', '.'), ('escapi/escapi_x64.dll', '.'), ('run.bat', '.'), ('msvcp140.dll', '.'), ('vcomp140.dll', '.'), ('concrt140.dll', '.'), ('vccorlib140.dll', '.')],
+             datas=[],
+             hiddenimports=[],
+             hookspath=[],
+             runtime_hooks=[],
+             excludes=['matplotlib', 'mpl-data', 'PyInstaller', 'pywt', 'skimage', 'scipy', 'pyinstaller'],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher,
+             noarchive=False)
+
+remove_bin = []
+for bin in a.binaries:
+    if bin[0].startswith("opencv_video") or bin[0].startswith("PyInstaller"):
+        remove_bin.append(bin[0])
+        print(bin)
+remove_dat = []
+for bin in a.datas:
+    if bin[0].startswith("opencv_video") or bin[0].startswith("PyInstaller"):
+        remove_dat.append(bin[0])
+        print("data ", bin)
+
+a.binaries = [x for x in a.binaries if not x[0] in remove_bin]
+a.datas = [x for x in a.datas if not x[0] in remove_dat]
+
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+exe = EXE(pyz,
+          a.scripts,
+          [],
+          exclude_binaries=True,
+          name='facetracker',
+          debug=False,
+          bootloader_ignore_signals=False,
+          strip=False,
+          upx=True,
+          console=True )
+
+coll = COLLECT(exe,
+               a.binaries,
+               a.zipfiles,
+               a.datas,
+               strip=False,
+               upx=True,
+               upx_exclude=[],
+               name='facetracker')

input_reader.py

@@ -2,18 +2,23 @@ import re
 import sys
 import os
 import cv2
-import numpy
+import numpy as np
 import escapi
-
+import dshowcapture
-def list_cameras():
+import time
-    print("Available cameras:")
+import traceback
+import gc
 
 class VideoReader():
-    def __init__(self, capture):
+    def __init__(self, capture, camera=False):
-        self.cap = cv2.VideoCapture(capture)
+        if os.name == 'nt' and camera:
+            self.cap = cv2.VideoCapture(capture, cv2.CAP_DSHOW)
+        else:
+            self.cap = cv2.VideoCapture(capture)
         if self.cap is None:
             print("The video source cannot be opened")
             sys.exit(0)
+        self.name = str(capture)
     def is_open(self):
         return self.cap.isOpened()
     def is_ready(self):
@@ -23,32 +28,22 @@ class VideoReader():
     def close(self):
         self.cap.release()
 
-class CameraReader(VideoReader):
+class EscapiReader(VideoReader):
     def __init__(self, capture, width, height, fps):
         self.device = None
         self.width = width
         self.height = height
         self.fps = fps
-        if os.name == 'nt':
+        self.device = capture
-            escapi.init()
+        escapi.count_capture_devices()
-            self.device = capture
+        self.name = str(escapi.device_name(self.device).decode('utf8', 'surrogateescape'))
-            self.buffer = escapi.init_camera(self.device, self.width, self.height, self.fps)
+        self.buffer = escapi.init_camera(self.device, self.width, self.height, self.fps)
-            escapi.do_capture(self.device)
+        escapi.do_capture(self.device)
-        else:
-            super(CameraReader, self).__init__(capture)
-            self.cap.set(3, width)
-            self.cap.set(4, height)
     def is_open(self):
-        if self.device is None:
-            return super(CameraReader, self).is_open()
         return True
     def is_ready(self):
-        if self.device is None:
-            return super(CameraReader, self).is_ready()
         return escapi.is_capture_done(self.device)
     def read(self):
-        if self.device is None:
-            return super(CameraReader, self).read()
         if escapi.is_capture_done(self.device):
             image = escapi.read(self.device, self.width, self.height, self.buffer)
             escapi.do_capture(self.device)
@@ -56,10 +51,99 @@ class CameraReader(VideoReader):
         else:
             return False, None
     def close(self):
-        if self.device is None:
+        escapi.deinit_camera(self.device)
-            super(CameraReader, self).close()
+
+class DShowCaptureReader(VideoReader):
+    def __init__(self, capture, width, height, fps, use_dshowcapture=True, dcap=None):
+        self.device = None
+        self.width = width
+        self.height = height
+        self.fps = fps
+        self.dcap = dcap;
+        self.device = dshowcapture.DShowCapture()
+        self.device.get_devices()
+        info = self.device.get_info()
+        self.name = info[capture]['name']
+        if info[capture]['type'] == "Blackmagic":
+            self.name = "Blackmagic: " + self.name
+            if dcap is None or dcap < 0:
+                dcap = 0
+        ret = False
+        if dcap is None:
+            ret = self.device.capture_device(capture, self.width, self.height, self.fps)
         else:
-            escapi.deinit_camera(self.device)
+            if dcap < 0:
+                ret = self.device.capture_device_default(capture)
+            else:
+                ret = self.device.capture_device_by_dcap(capture, dcap, self.width, self.height, self.fps)
+        if not ret:
+            raise Exception("Failed to start capture.")
+        self.width = self.device.width
+        self.height = self.device.height
+        self.fps = self.device.fps
+        print(f"Camera: \"{self.name}\" Capability ID: {dcap} Resolution: {self.device.width}x{self.device.height} Frame rate: {self.device.fps} Colorspace: {self.device.colorspace} Internal: {self.device.colorspace_internal} Flipped: {self.device.flipped}")
+        self.timeout = 1000
+    def is_open(self):
+        return self.device.capturing()
+    def is_ready(self):
+        return self.device.capturing()
+    def read(self):
+        img = None
+        try:
+            img = self.device.get_frame(self.timeout)
+        except:
+            gc.collect()
+            img = self.device.get_frame(self.timeout)
+        if img is None:
+            return False, None
+        else:
+            return True, img
+    def close(self):
+        self.device.destroy_capture()
+
+class OpenCVReader(VideoReader):
+    def __init__(self, capture, width, height, fps):
+        self.device = None
+        self.width = width
+        self.height = height
+        self.fps = fps
+        self.name = str(capture)
+        super(OpenCVReader, self).__init__(capture, camera=True)
+        self.cap.set(3, width)
+        self.cap.set(4, height)
+        self.cap.set(38, 1)
+    def is_open(self):
+        return super(OpenCVReader, self).is_open()
+    def is_ready(self):
+        return super(OpenCVReader, self).is_ready()
+    def read(self):
+        return super(OpenCVReader, self).read()
+    def close(self):
+        super(OpenCVReader, self).close()
+
+class V4L2Reader(OpenCVReader):
+    def __init__(self, capture, width, height, fps, dformat):
+        self.cap = cv2.VideoCapture(capture, cv2.CAP_V4L2)
+        if dformat:
+            fourcc = cv2.VideoWriter.fourcc(*dformat)
+            self.cap.set(6, fourcc)
+        self.device = None
+        self.width = width
+        self.height = height
+        self.fps = fps
+        self.name = str(capture)
+        self.cap.set(3, width)
+        self.cap.set(4, height)
+        self.cap.set(38, 1)
+        self.cap.set(5, fps)
+    def is_open(self):
+        return super(V4L2Reader, self).is_open()
+    def is_ready(self):
+        return super(V4L2Reader, self).is_ready()
+    def read(self):
+        return super(V4L2Reader, self).read()
+    def close(self):
+        super(V4L2Reader, self).close()
 
 class RawReader:
     def __init__(self, width, height):
@@ -83,20 +167,105 @@ class RawReader:
             bytes = sys.stdin.buffer.read(self.len)
             read_bytes += len(bytes)
             frame.extend(bytes)
-        return np.frombuffer(frame, dtype=np.uint8).reshape((self.height, self.width, 3))
+        return True, np.frombuffer(frame, dtype=np.uint8).reshape((self.height, self.width, 3))
     def close(self):
         self.open = False
 
-class InputReader():
+def try_int(s):
-    def __init__(self, capture, raw_rgb, width, height, fps):
+    try:
-        self.reader = None
+        return int(s)
-        try:
+    except:
-            if os.path.exists(capture):
+        return None
-                self.reader = VideoReader(capture)
+
-            elif capture == str(int(capture)):
+def test_reader(reader):
-                self.reader = CameraReader(int(capture), width, height, fps)
+    got_any = 0
+    try:
+        for i in range(30):
+            if not reader.is_ready():
+                time.sleep(0.02)
+            ret, frame = reader.read()
+            if not ret:
+                time.sleep(0.02)
+                print("No frame")
             else:
+                print("Got frame")
+                got_any += 1
+                if got_any > 10:
+                    break
+        if reader.is_open():
+            return got_any > 0
+        print("Fail")
+        return False
+    except:
+        traceback.print_exc()
+        print("Except")
+        return False
+
+class InputReader():
+    def __init__(self, capture, raw_rgb, width, height, fps, use_dshowcapture=False, dcap=None):
+        self.reader = None
+        self.name = str(capture)
+        try:
+            if raw_rgb > 0:
                 self.reader = RawReader(width, height)
+            elif os.path.exists(capture):
+                self.reader = VideoReader(capture)
+            elif capture == str(try_int(capture)):
+                if os.name == 'nt':
+                    # Try with DShowCapture
+                    good = True
+                    name = ""
+                    try:
+                        if use_dshowcapture:
+                            self.reader = DShowCaptureReader(int(capture), width, height, fps, dcap=dcap)
+                            name = self.reader.name
+                            good = test_reader(self.reader)
+                            self.name = name
+                        else:
+                            good = False
+                    except:
+                        print("DShowCapture exception: ")
+                        traceback.print_exc()
+                        good = False
+                    if good:
+                        return
+                    # Try with Escapi
+                    good = True
+                    try:
+                        print(f"DShowCapture failed. Falling back to escapi for device {name}.", file=sys.stderr)
+                        escapi.init()
+                        devices = escapi.count_capture_devices()
+                        found = None
+                        for i in range(devices):
+                            escapi_name = str(escapi.device_name(i).decode('utf8', 'surrogateescape'))
+                            if name == escapi_name:
+                                found = i
+                        if found is None:
+                            good = False
+                        else:
+                            print(f"Found device {name} as {i}.", file=sys.stderr)
+                            self.reader = EscapiReader(found, width, height, fps)
+                            good = test_reader(self.reader)
+                    except:
+                        print("Escapi exception: ")
+                        traceback.print_exc()
+                        good = False
+                    if good:
+                        return
+                    # Try with OpenCV
+                    print(f"Escapi failed. Falling back to OpenCV. If this fails, please change your camera settings.", file=sys.stderr)
+                    self.reader = OpenCVReader(int(capture), width, height, fps)
+                    self.name = self.reader.name
+                elif sys.platform == 'linux':
+                    try:
+                        self.reader = V4L2Reader(int(capture), width, height, fps, dcap)
+                    except:
+                        print("V4L2 exception: ")
+                        traceback.print_exc()
+                        print(f"V4L2 failed. Falling back to OpenCV. If this fails, please change your camera settings.", file=sys.stderr)
+                        self.reader = OpenCVReader(int(capture), width, height, fps)
+                else:
+                    self.reader = OpenCVReader(int(capture), width, height, fps)
         except Exception as e:
             print("Error: " + str(e))
 
@@ -110,4 +279,4 @@ class InputReader():
     def read(self):
         return self.reader.read()
     def close(self):
-        self.reader.close()
+        self.reader.close()

make_exe.bat

@@ -1,9 +1,31 @@
-REM pyinstaller --onefile ^
+echo "Started"
-REM     --add-binary escapi/escapi_x86.dll;. ^
-REM     --add-binary escapi/escapi_x64.dll;. ^
-REM     facetracker.py
 
-pyinstaller --onedir ^
+:: A venv is used so it's easy to add the onnxruntime dll to the binary
-    --add-binary escapi/escapi_x86.dll;. ^
+
-    --add-binary escapi/escapi_x64.dll;. ^
+echo "Creating venv"
-    facetracker.py
+python -m venv venv
+
+echo "Activating venv"
+call venv\Scripts\activate.bat
+
+echo "Installing dependencies"
+pip install wheel
+
+pip install onnxruntime opencv-python==4.5.4.60 pillow numpy==1.23.0 pyinstaller
+
+echo "Running pyinstaller"
+pyinstaller facetracker.py --clean ^
+    --onedir ^
+    --add-binary dshowcapture/*.dll;. ^
+    --add-binary escapi/*.dll;. ^
+    --add-binary venv/lib/site-packages/onnxruntime/capi/*.dll;onnxruntime\capi ^
+    --add-binary msvcp140.dll;. ^
+    --add-binary vcomp140.dll;. ^
+    --add-binary concrt140.dll;. ^
+    --add-binary vccorlib140.dll;. ^
+    --add-binary run.bat;.
+
+echo "Deleting opencv dll"
+del dist\facetracker\cv2\opencv_videoio_*
+
+echo "Finished"

make_exe.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+
+PYTHON_BINARY=$(which python3)
+
+if [[ -z "$PYTHON_BINARY" ]]; then
+    echo "python3 not found. Searching for python instead."
+    PYTHON_BINARY=$(which python)
+    if [[ -z "$PYTHON_BINARY" ]]; then
+        echo "Python 3 must be available on your path. Exiting."
+        exit 1
+    fi
+fi
+
+echo "Creating venv"
+"$PYTHON_BINARY" -m venv venv
+
+source venv/bin/activate
+
+echo "Installing packages"
+pip install wheel # Make sure this is installed beforehand
+
+pip install onnxruntime opencv-python==4.5.4.60 pillow numpy==1.23.0 pyinstaller
+
+echo "Creating binary"
+pyinstaller --onedir --clean facetracker.py \
+    --add-binary venv/lib/python3.*/site-packages/onnxruntime/capi/*.so:onnxruntime/capi
+
+echo "Files should be available in the dist/ folder"
+echo "Done!"
+

model.py

@@ -1,164 +1,225 @@
-# This file is not used by the tracking application
+# This file is not used by the tracking application and currently outdated
 import torch
 import torch.nn as nn
+import geffnet.mobilenetv3 # geffnet.mobilenetv3._gen_mobilenet_v3 needs to be patched to return the parameters instead of instantiating the network, based on commit c450c12ae6ffb1757f62dde3c2765da3c10f6def
+from geffnet.efficientnet_builder import round_channels
 
 class DSConv2d(nn.Module):
-    def __init__(self, in_planes, out_planes, kernels_per_layer=4, groups=1):
+    def __init__(self, in_planes, out_planes, kernels_per_layer=4, groups=1, old=0):
         super(DSConv2d, self).__init__()
-        self.conv = nn.Sequential(
+        if old == 2:
-            nn.Conv2d(in_planes, in_planes * kernels_per_layer, kernel_size=3, padding=1, groups=in_planes),
+            self.conv = nn.Sequential(
-            nn.Conv2d(in_planes * kernels_per_layer, out_planes, kernel_size=1, groups=groups)
+                nn.Conv2d(in_planes, in_planes * kernels_per_layer, kernel_size=3, padding=1, groups=in_planes),
-        )
+                nn.Conv2d(in_planes * kernels_per_layer, out_planes, kernel_size=1, groups=groups)
+            )
+        elif old == 1:
+            self.conv = nn.Sequential(
+                nn.Conv2d(in_planes, in_planes * kernels_per_layer, kernel_size=3, padding=1, groups=in_planes, bias=False),
+                nn.BatchNorm2d(in_planes * kernels_per_layer),
+                nn.Conv2d(in_planes * kernels_per_layer, out_planes, kernel_size=1, groups=groups, bias=False),
+                nn.BatchNorm2d(out_planes),
+                nn.ReLU6(inplace=True)
+            )
+        else:
+            self.conv = nn.Sequential(
+                nn.Conv2d(in_planes, in_planes * kernels_per_layer, kernel_size=3, padding=1, groups=in_planes, bias=False),
+                nn.BatchNorm2d(in_planes * kernels_per_layer),
+                nn.ReLU6(inplace=True),
+                nn.Conv2d(in_planes * kernels_per_layer, out_planes, kernel_size=1, groups=groups, bias=False),
+                nn.BatchNorm2d(out_planes),
+                nn.ReLU6(inplace=True)
+            )
     def forward(self, x):
         x = self.conv(x)
         return x
 
 class UNetUp(nn.Module):
-    def __init__(self, in_channels, residual_in_channels, out_channels, size):
+    def __init__(self, in_channels, residual_in_channels, out_channels, size, old=0):
         super(UNetUp, self).__init__()
         self.up = nn.Upsample(size=size, mode='bilinear', align_corners=True)
-        self.conv = DSConv2d(in_channels + residual_in_channels, out_channels, 1, 1)
+        self.conv = DSConv2d(in_channels + residual_in_channels, out_channels, 1, 1, old=old)
     def forward(self, x1, x2):
         x1 = self.up(x1)
         x = torch.cat([x2, x1], dim=1)
         x = self.conv(x)
         return x
 
-# Copy of torchvision ShuffleNetV2 for type annotation
+# This is the gaze tracking model
-def channel_shuffle(x, groups: int):
+class OpenSeeFaceGaze(geffnet.mobilenetv3.MobileNetV3):
-    batchsize, num_channels, height, width = x.data.size()
-    channels_per_group = num_channels // groups
-
-    # reshape
-    x = x.view(batchsize, groups,
-               channels_per_group, height, width)
-
-    x = torch.transpose(x, 1, 2).contiguous()
-
-    # flatten
-    x = x.view(batchsize, -1, height, width)
-
-    return x
-
-class InvertedResidual(nn.Module):
-    def __init__(self, inp, oup, stride):
-        super(InvertedResidual, self).__init__()
-
-        if not (1 <= stride <= 3):
-            raise ValueError('illegal stride value')
-        self.stride = stride
-
-        branch_features = oup // 2
-        assert (self.stride != 1) or (inp == branch_features << 1)
-
-        self.branch1 = nn.Sequential(
-            self.depthwise_conv(inp, inp, kernel_size=3, stride=self.stride, padding=1),
-            nn.BatchNorm2d(inp),
-            nn.Conv2d(inp, branch_features, kernel_size=1, stride=1, padding=0, bias=False),
-            nn.BatchNorm2d(branch_features),
-            nn.ReLU(inplace=True),
-        )
-
-        self.branch2 = nn.Sequential(
-            nn.Conv2d(inp if (self.stride > 1) else branch_features,
-                      branch_features, kernel_size=1, stride=1, padding=0, bias=False),
-            nn.BatchNorm2d(branch_features),
-            nn.ReLU(inplace=True),
-            self.depthwise_conv(branch_features, branch_features, kernel_size=3, stride=self.stride, padding=1),
-            nn.BatchNorm2d(branch_features),
-            nn.Conv2d(branch_features, branch_features, kernel_size=1, stride=1, padding=0, bias=False),
-            nn.BatchNorm2d(branch_features),
-            nn.ReLU(inplace=True),
-        )
-
-    @staticmethod
-    def depthwise_conv(i, o, kernel_size, stride=1, padding=0, bias=False):
-        return nn.Conv2d(i, o, kernel_size, stride, padding, bias=bias, groups=i)
-
-    def forward(self, x):
-        if self.stride == 1:
-            x1, x2 = x.chunk(2, dim=1)
-            out = torch.cat((x1, self.branch2(x2)), dim=1)
-        else:
-            out = torch.cat((self.branch1(x), self.branch2(x)), dim=1)
-
-        out = channel_shuffle(out, 2)
-
-        return out
-
-class ShuffleNetV2(nn.Module):
-    def __init__(self, stages_repeats, stages_out_channels, num_classes=1000):
-        super(ShuffleNetV2, self).__init__()
-
-        if len(stages_repeats) != 3:
-            raise ValueError('expected stages_repeats as list of 3 positive ints')
-        if len(stages_out_channels) != 5:
-            raise ValueError('expected stages_out_channels as list of 5 positive ints')
-        self._stage_out_channels = stages_out_channels
-
-        input_channels = 3
-        output_channels = self._stage_out_channels[0]
-        self.conv1 = nn.Sequential(
-            nn.Conv2d(input_channels, output_channels, 3, 2, 1, bias=False),
-            nn.BatchNorm2d(output_channels),
-            nn.ReLU(inplace=True),
-        )
-        input_channels = output_channels
-
-        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
-
-        stage_names = ['stage{}'.format(i) for i in [2, 3, 4]]
-        for name, repeats, output_channels in zip(
-                stage_names, stages_repeats, self._stage_out_channels[1:]):
-            seq = [InvertedResidual(input_channels, output_channels, 2)]
-            for i in range(repeats - 1):
-                seq.append(InvertedResidual(output_channels, output_channels, 1))
-            setattr(self, name, nn.Sequential(*seq))
-            input_channels = output_channels
-
-        output_channels = self._stage_out_channels[-1]
-        self.conv5 = nn.Sequential(
-            nn.Conv2d(input_channels, output_channels, 1, 1, 0, bias=False),
-            nn.BatchNorm2d(output_channels),
-            nn.ReLU(inplace=True),
-        )
-
-        self.fc = nn.Linear(output_channels, num_classes)
-
-    def forward(self, x):
-        x = self.conv1(x)
-        x = self.maxpool(x)
-        x = self.stage2(x)
-        x = self.stage3(x)
-        x = self.stage4(x)
-        x = self.conv5(x)
-        x = x.mean([2, 3])  # globalpool
-        x = self.fc(x)
-        return x
-
-# Facial landmark detection model
-class OpenSeeNetSNV2(ShuffleNetV2):
     def __init__(self):
-        super(OpenSeeNetSNV2, self).__init__([4, 8, 4], [24, 116, 232, 464, 1024])
+        kwargs = geffnet.mobilenetv3._gen_mobilenet_v3(['small'])
-        self.up1 = UNetUp(1024, 232, 256, (14,14))
+        super(OpenSeeFaceGaze, self).__init__(**kwargs)
-        self.up2 = UNetUp(256, 116, 198 * 1, (28,28))
+        self.up1 = UNetUp(576, 48, 64, (2,2), old=2)
-        self.group = DSConv2d(198 * 1, 198, kernels_per_layer=4, groups=3)
+        self.up2 = UNetUp(64, 24, 32, (4,4), old=2)
-
+        self.up3 = UNetUp(32, 16, 15, (8,8), old=2)
+        self.group = DSConv2d(15, 3, kernels_per_layer=4, groups=3, old=2)
     def _forward_impl(self, x):
-        x = self.conv1(x)
+        x = self.conv_stem(x)
-        x = self.maxpool(x)
+        x = self.bn1(x)
-        x = self.stage2(x)
+        x = self.act1(x)
-        r2 = x
+        r1 = None
-        x = self.stage3(x)
+        r2 = None
-        r3 = x
+        r3 = None
-        x = self.stage4(x)
+        for i, feature in enumerate(self.blocks):
-        x = self.conv5(x)
+            x = feature(x)
+            if i == 3:
+                r3 = x
+            if i == 1:
+                r2 = x
+            if i == 0:
+                r1 = x
+        x = self.up1(x, r3)
+        x = self.up2(x, r2)
+        x = self.up3(x, r1)
+        x = self.group(x)
+        return x
+    def forward(self, x):
+        return self._forward_impl(x)
 
+# This is the face detection model. Because the landmark model is very robust, it gets away with predicting very rough bounding boxes. It is fully convolutional and can be made to run on different resolutions. It was trained on 224x224 crops and the most reasonable results can be found in the range of 224x224 to 640x640.
+class OpenSeeFaceDetect(geffnet.mobilenetv3.MobileNetV3):
+    def __init__(self, size="large", channel_multiplier=0.1):
+        kwargs = geffnet.mobilenetv3._gen_mobilenet_v3([size], channel_multiplier=channel_multiplier)
+        super(OpenSeeFaceDetect, self).__init__(**kwargs)
+        if size == "large":
+            self.up1 = UNetUp(round_channels(960, channel_multiplier), round_channels(112, channel_multiplier), 256, (14,14), old=1)
+            self.up2 = UNetUp(256, round_channels(40, channel_multiplier), 128, (28,28), old=1)
+            self.up3 = UNetUp(128, round_channels(24, channel_multiplier), 64, (56,56), old=1)
+            self.group = DSConv2d(64, 2, kernels_per_layer=4, groups=2, old=1)
+            self.r1_i = 1
+            self.r2_i = 2
+            self.r3_i = 4
+        elif size == "small":
+            self.up1 = UNetUp(round_channels(576, channel_multiplier), round_channels(40, channel_multiplier), 256, (14,14), old=1)
+            self.up2 = UNetUp(256, round_channels(24, channel_multiplier), 128, (28,28), old=1)
+            self.up3 = UNetUp(128, round_channels(16, channel_multiplier), 64, (56,56), old=1)
+            self.group = DSConv2d(64, 2, kernels_per_layer=4, groups=2, old=1)
+            self.r1_i = 0
+            self.r2_i = 1
+            self.r3_i = 2
+        self.maxpool = nn.MaxPool2d(kernel_size=3, dilation=1, stride=1, padding=1)
+    def _forward_impl(self, x):
+        x = self.conv_stem(x)
+        x = self.bn1(x)
+        x = self.act1(x)
+        r2 = None
+        r3 = None
+        for i, feature in enumerate(self.blocks):
+            x = feature(x)
+            if i == self.r3_i:
+                r3 = x
+            if i == self.r2_i:
+                r2 = x
+            if i == self.r1_i:
+                r1 = x
+        x = self.up1(x, r3)
+        x = self.up2(x, r2)
+        x = self.up3(x, r1)
+        x = self.group(x)
+        x2 = self.maxpool(x)
+        return x, x2
+    def forward(self, x):
+        return self._forward_impl(x)
+
+def logit_arr(p, factor=16.0):
+    p = p.clamp(0.0000001, 0.9999999)
+    return torch.log(p / (1 - p)) / factor
+
+# Landmark detection model
+# Models:
+# 0: "small", 0.5
+# 1: "small", 1.0
+# 2: "large", 0.75
+# 3: "large", 1.0
+class OpenSeeFaceLandmarks(geffnet.mobilenetv3.MobileNetV3):
+    def __init__(self, size="large", channel_multiplier=1.0, inference=False):
+        kwargs = geffnet.mobilenetv3._gen_mobilenet_v3([size], channel_multiplier=channel_multiplier)
+        super(OpenSeeFaceLandmarks, self).__init__(**kwargs)
+        if size == "large":
+            self.up1 = UNetUp(round_channels(960, channel_multiplier), round_channels(112, channel_multiplier), 256, (14,14))
+            self.up2 = UNetUp(256, round_channels(40, channel_multiplier), 198 * 1, (28,28))
+            self.group = DSConv2d(198 * 1, 198, kernels_per_layer=4, groups=3)
+            self.r2_i = 2
+            self.r3_i = 4
+        elif size == "small":
+            self.up1 = UNetUp(round_channels(576, channel_multiplier), round_channels(40, channel_multiplier), 256, (14,14))
+            self.up2 = UNetUp(256, round_channels(24, channel_multiplier), 198 * 1, (28,28))
+            self.group = DSConv2d(198 * 1, 198, kernels_per_layer=4, groups=3)
+            self.r2_i = 1
+            self.r3_i = 2
+        self.inference = inference
+    def _forward_impl(self, x):
+        x = self.conv_stem(x)
+        x = self.bn1(x)
+        x = self.act1(x)
+        r2 = None
+        r3 = None
+        for i, feature in enumerate(self.blocks):
+            x = feature(x)
+            if i == self.r3_i:
+                r3 = x
+            if i == self.r2_i:
+                r2 = x
         x = self.up1(x, r3)
         x = self.up2(x, r2)
         x = self.group(x)
-        return x
 
+        if self.inference:
+            t_main = x[:, 0:66].reshape((-1, 66, 28*28))
+            t_m = t_main.argmax(dim=2)
+            indices = t_m.unsqueeze(2)
+            t_conf = t_main.gather(2, indices).squeeze(2)
+            t_off_x = x[:, 66:132].reshape((-1, 66, 28*28)).gather(2, indices).squeeze(2)
+            t_off_y = x[:, 132:198].reshape((-1, 66, 28*28)).gather(2, indices).squeeze(2)
+            t_off_x = (223. * logit_arr(t_off_x) + 0.5).floor()
+            t_off_y = (223. * logit_arr(t_off_y) + 0.5).floor()
+            t_x = 223. * (t_m / 28.).floor() / 27. + t_off_x
+            t_y = 223. * t_m.remainder(28.).float() / 27. + t_off_y
+            x = (t_conf.mean(1), torch.stack([t_x, t_y, t_conf], 2))
+
+        return x
+    def forward(self, x):
+        return self._forward_impl(x)
+
+# lm_modelT for 56x56 30 point inference
+class OpenSeeFaceLandmarks30Pt(geffnet.mobilenetv3.MobileNetV3):
+    def __init__(self, size="large", channel_multiplier=1.0, inference=False):
+        kwargs = geffnet.mobilenetv3._gen_mobilenet_v3([size], channel_multiplier=channel_multiplier)
+        super(OpenSeeFaceLandmarks30Pt, self).__init__(**kwargs)
+        self.up1 = UNetUp(960, 112, 256, (4,4))
+        self.up2 = UNetUp(256, 40, 180, (7,7))
+        self.group = DSConv2d(180, 90, kernels_per_layer=4, groups=3)
+        self.inference = inference
+    def _forward_impl(self, x):
+        x = self.conv_stem(x)
+        x = self.bn1(x)
+        x = self.act1(x)
+        r2 = None
+        r3 = None
+        for i, feature in enumerate(self.blocks):
+            x = feature(x)
+            if i == 4:
+                r3 = x
+            if i == 2:
+                r2 = x
+        x = self.up1(x, r3)
+        x = self.up2(x, r2)
+        x = self.group(x)
+
+        if self.inference:
+            t_main = x[:, 0:30].reshape((-1, 30, 7*7))
+            t_m = t_main.argmax(dim=2)
+            indices = t_m.unsqueeze(2)
+            t_conf = t_main.gather(2, indices).squeeze(2)
+            t_off_x = x[:, 30:60].reshape((-1, 30, 7*7)).gather(2, indices).squeeze(2)
+            t_off_y = x[:, 60:90].reshape((-1, 30, 7*7)).gather(2, indices).squeeze(2)
+            t_off_x = 55. * logit_arr(t_off_x, factor=8.0)
+            t_off_y = 55. * logit_arr(t_off_y, factor=8.0)
+            t_x = 55. * (t_m / 7.).floor() / 6. + t_off_x
+            t_y = 55. * t_m.remainder(7.).float() / 6. + t_off_y
+            x = (t_conf.mean(1), torch.stack([t_x, t_y, t_conf], 2))
+
+        return x
     def forward(self, x):
         return self._forward_impl(x)
 
@@ -190,18 +251,21 @@ def AdapWingLoss(pre_hm, gt_hm):
         img_dilate = cv2.morphologyEx(img_merge, cv2.MORPH_DILATE, kernel)
         img_dilate[img_dilate < 51] = 1  # 0*W+1
         img_dilate[img_dilate >= 51] = 11  # 1*W+1
-        img_dilate = np.array(img_dilate, dtype=np.int)
+        img_dilate = np.array(img_dilate, dtype=int)
         img_dilate = img_dilate.transpose(2, 0, 1)
         dilated = torch.from_numpy(img_dilate).float()
         if first:
             first_mask = dilated
         first = False
-        dilated[17:27] *= 1.2
+        # These weights varied between training runs and model sizes
-        dilated[17] *= 1.3
+        dilated[17:27][dilated[17:27] > 1] *= 1.4
-        dilated[18] *= 1.4
+        dilated[17, dilated[17] > 1] *= 1.6
-        dilated[25] *= 1.4
+        dilated[18, dilated[18] > 1] *= 1.8
-        dilated[26] *= 1.3
+        dilated[25, dilated[25] > 1] *= 1.8
-        dilated[36:47] *= 2.5
+        dilated[26, dilated[26] > 1] *= 1.6
+        dilated[36:48][dilated[36:48] > 1] *= 2.8
+        # Used for a very small model
+        #dilated[[37,38,40,41,43,44,46,47]][dilated[[37,38,40,41,43,44,46,47]] > 1] *= 20.8
         mask[i] = torch.cat([dilated, dilated, dilated], 0)
 
     diff_hm = torch.abs(gt_hm - pre_hm)
@@ -216,5 +280,35 @@ def AdapWingLoss(pre_hm, gt_hm):
 
     return first_mask.detach(), mean_loss
 
-# The fast model runs on grayscale 112x112 using the ShuffleNet V2 0.5x configuration with input_channels=1. It was trained using AWL, but without the additional weighting for certain features
+
-.
+
+# Checkpoint test
+if __name__== "__main__":
+    print("Checking gaze model")
+    m=OpenSeeFaceGaze()
+    ckpt = torch.load("gaze.pth")
+    m.load_state_dict(ckpt)
+    print("Checking detection model")
+    m=OpenSeeFaceDetect()
+    ckpt = torch.load("detection.pth")
+    m.load_state_dict(ckpt)
+    print("Checking lm_model0 model")
+    m=OpenSeeFaceLandmarks("small", 0.5)
+    ckpt = torch.load("lm_model0.pth")
+    m.load_state_dict(ckpt)
+    print("Checking lm_model1 model")
+    m=OpenSeeFaceLandmarks("small", 1.0)
+    ckpt = torch.load("lm_model1.pth")
+    m.load_state_dict(ckpt)
+    print("Checking lm_model2 model")
+    m=OpenSeeFaceLandmarks("large", 0.75)
+    ckpt = torch.load("lm_model2.pth")
+    m.load_state_dict(ckpt)
+    print("Checking lm_model3 model")
+    m=OpenSeeFaceLandmarks("large", 1.0)
+    ckpt = torch.load("lm_model3.pth")
+    m.load_state_dict(ckpt)
+    print("Checking lm_modelT model")
+    m=OpenSeeFaceLandmarks("large", 1.0)
+    ckpt = torch.load("lm_modelT.pth")
+    m.load_state_dict(ckpt)