From 633293eb52d10a052567be6ff92cbf7b59a1e3ff Mon Sep 17 00:00:00 2001 From: Emiliana <38952746+emilianavt@users.noreply.github.com> Date: Sun, 16 Feb 2020 15:58:47 +0100 Subject: [PATCH] The face tracker can now offset face ids by a given value. This makes it possible to easily mix tracking input from multiple network sources. --- Unity/OpenSee.cs | 33 ++++++++++++++++++++++++++------- Unity/OpenSeeExpression.cs | 36 +++++++++--------------------------- Unity/OpenSeeIKTarget.cs | 21 ++++++--------------- Unity/OpenSeeLauncher.cs | 2 +- Unity/OpenSeeShowPoints.cs | 32 ++++++++++++++++++-------------- facetracker.py | 4 ++++ 6 files changed, 64 insertions(+), 64 deletions(-) diff --git a/Unity/OpenSee.cs b/Unity/OpenSee.cs index f3bfab3..6f374e4 100644 --- a/Unity/OpenSee.cs +++ b/Unity/OpenSee.cs @@ -85,7 +85,7 @@ public class OpenSee : MonoBehaviour { Vector2 v = new Vector2(readFloat(b, ref o), readFloat(b, ref o)); return v; } - + public void readFromPacket(byte[] b, int o) { time = System.BitConverter.ToDouble(b, o); o += 8; @@ -139,11 +139,20 @@ public class OpenSee : MonoBehaviour { } } + private Dictionary openSeeDataMap; private Socket socket; private byte[] buffer; private Thread receiveThread = null; private volatile bool stopReception = false; + public OpenSeeData GetOpenSeeData(int faceId) { + if (openSeeDataMap == null) + return null; + if (!openSeeDataMap.ContainsKey(faceId)) + return null; + return openSeeDataMap[faceId]; + } + void performReception() { EndPoint senderRemote = new IPEndPoint(IPAddress.Any, 0); while (!stopReception) { @@ -154,18 +163,20 @@ public class OpenSee : MonoBehaviour { } receivedPackets++; int i = 0; - OpenSeeData[] newData = new OpenSeeData[receivedBytes / packetFrameSize]; for (int offset = 0; offset < receivedBytes; offset += packetFrameSize) { - newData[i] = new OpenSeeData(); - newData[i].readFromPacket(buffer, offset); + OpenSeeData newData = new OpenSeeData(); + newData.readFromPacket(buffer, offset); + openSeeDataMap[newData.id] = newData; i++; } - trackingData = newData; + trackingData = new OpenSeeData[openSeeDataMap.Count]; + openSeeDataMap.Values.CopyTo(trackingData, 0); } catch { } } } void Start () { + openSeeDataMap = new Dictionary(); buffer = new byte[65535]; socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); @@ -180,8 +191,8 @@ public class OpenSee : MonoBehaviour { void Update () { } - - void OnApplicationQuit() { + + void EndReceiver() { if (receiveThread != null) { stopReception = true; receiveThread.Join(); @@ -190,6 +201,14 @@ public class OpenSee : MonoBehaviour { if (socket != null) socket.Close(); } + + void OnApplicationQuit() { + EndReceiver(); + } + + void OnDestroy() { + EndReceiver(); + } } } \ No newline at end of file diff --git a/Unity/OpenSeeExpression.cs b/Unity/OpenSeeExpression.cs index 50ce3e9..19ccb99 100644 --- a/Unity/OpenSeeExpression.cs +++ b/Unity/OpenSeeExpression.cs @@ -137,6 +137,7 @@ public class OpenSeeExpression : MonoBehaviour } } + private OpenSee.OpenSeeData openSeeData = null; private Dictionary> expressions; private SVMModel model = null; private string[] classLabels = null; @@ -394,16 +395,10 @@ public class OpenSeeExpression : MonoBehaviour ResetInfo(); return false; } - OpenSee.OpenSeeData[] openSeeData = openSee.trackingData; - if (openSeeData == null) + openSeeData = openSee.GetOpenSeeData(faceId); + if (openSeeData == null || openSeeData.time <= lastCapture) return false; - OpenSee.OpenSeeData t = null; - foreach (OpenSee.OpenSeeData data in openSeeData) - if (data.id == faceId) - t = data; - if (t == null || t.time <= lastCapture) - return false; - float[] faceData = GetData(t); + float[] faceData = GetData(openSeeData); float[] predictionData = new float[cols]; for (int i = 0; i < cols; i++) predictionData[i] = faceData[indices[i]]; @@ -436,14 +431,8 @@ public class OpenSeeExpression : MonoBehaviour if (calibrationExpression == "") return false; if (expressions.ContainsKey(calibrationExpression) || expressions.Keys.Count < 10) { - OpenSee.OpenSeeData[] openSeeData = openSee.trackingData; - if (openSeeData == null) - return false; - OpenSee.OpenSeeData t = null; - foreach (OpenSee.OpenSeeData data in openSeeData) - if (data.id == faceId) - t = data; - if (t == null || t.time <= lastCapture) + openSeeData = openSee.GetOpenSeeData(faceId); + if (openSeeData == null || openSeeData.time <= lastCapture) return false; if (!expressions.ContainsKey(calibrationExpression)) { expressions.Add(calibrationExpression, new List()); @@ -461,9 +450,9 @@ public class OpenSeeExpression : MonoBehaviour if (frameCount % skip != 0) return true; if (percentRecorded >= 100f) { - list[rnd.Next(maxSamples)] = GetData(t); + list[rnd.Next(maxSamples)] = GetData(openSeeData); } else - list.Add(GetData(t)); + list.Add(GetData(openSeeData)); percentRecorded = 100f * (float)list.Count/(float)maxSamples; return true; } else { @@ -566,16 +555,9 @@ public class OpenSeeExpression : MonoBehaviour else percentRecorded = 0f; - OpenSee.OpenSeeData[] openSeeData = openSee.trackingData; if (openSeeData == null) return; - OpenSee.OpenSeeData t = null; - foreach (OpenSee.OpenSeeData data in openSeeData) - if (data.id == faceId) - t = data; - if (t == null || t.time <= lastCapture) - return; - lastCapture = t.time; + lastCapture = openSeeData.time; } } diff --git a/Unity/OpenSeeIKTarget.cs b/Unity/OpenSeeIKTarget.cs index e5871f6..a449034 100644 --- a/Unity/OpenSeeIKTarget.cs +++ b/Unity/OpenSeeIKTarget.cs @@ -61,27 +61,18 @@ public class OpenSeeIKTarget : MonoBehaviour } void RunUpdate() { - var openSeeData = openSee.trackingData; - if (openSeeData == null || openSeeData.Length < 1) + var openSeeData = openSee.GetOpenSeeData(faceId); + if (openSeeData == null) return; - int idx = -1; - int i = 0; - foreach (var item in openSeeData) { - if (item.id == faceId) - idx = i; - i++; - } - if (idx < 0) - return; - if (openSeeData[idx].time > updated) { - updated = openSeeData[idx].time; + if (openSeeData.time > updated) { + updated = openSeeData.time; } else { Interpolate(); return; } - Quaternion convertedQuaternion = new Quaternion(-openSeeData[idx].rawQuaternion.y, -openSeeData[idx].rawQuaternion.x, openSeeData[idx].rawQuaternion.z, openSeeData[idx].rawQuaternion.w); - Vector3 t = openSeeData[idx].translation; + Quaternion convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w); + Vector3 t = openSeeData.translation; t.x = -t.x; t.z = -t.z; diff --git a/Unity/OpenSeeLauncher.cs b/Unity/OpenSeeLauncher.cs index 50034c8..1b54f72 100644 --- a/Unity/OpenSeeLauncher.cs +++ b/Unity/OpenSeeLauncher.cs @@ -29,7 +29,7 @@ public class OpenSeeLauncher : MonoBehaviour { [Tooltip("The path, where the .onnx model files can be found")] public string modelPath = "models\\"; [Tooltip("Additional options that should be passed to the face tracker.")] - public List commandlineOptions = new List(new string[] { "--silent", "1", "--max-threads", "2", "--model", "3" }); + public List commandlineOptions = new List(new string[] { "--silent", "1", "--max-threads", "4" }); [Header("Runtime information")] [Tooltip("This field shows if the tracker is currently alive.")] public bool trackerAlive = false; diff --git a/Unity/OpenSeeShowPoints.cs b/Unity/OpenSeeShowPoints.cs index b96d2e6..2b131bf 100644 --- a/Unity/OpenSeeShowPoints.cs +++ b/Unity/OpenSeeShowPoints.cs @@ -6,12 +6,13 @@ namespace OpenSee { public class OpenSeeShowPoints : MonoBehaviour { public OpenSee openSee = null; + public int faceId = 0; public bool show3DPoints = true; public bool applyTranslation = false; public bool applyRotation = false; [Range(0, 1)] public float minConfidence = 0.20f; - private OpenSee.OpenSeeData[] openSeeData; + private OpenSee.OpenSeeData openSeeData; private GameObject[] gameObjects; private GameObject centerBall; private double updated = 0.0; @@ -36,21 +37,24 @@ public class OpenSeeShowPoints : MonoBehaviour { void Update () { if (!openSee) return; - openSeeData = openSee.trackingData; + /*openSeeData = openSee.trackingData; if (openSeeData == null || openSeeData.Length < 1) + return;*/ + openSeeData = openSee.GetOpenSeeData(faceId); + if (openSeeData == null) return; - if (openSeeData[0].time > updated) { - updated = openSeeData[0].time; + if (openSeeData.time > updated) { + updated = openSeeData.time; } else { return; } if (show3DPoints) { centerBall.gameObject.SetActive(false); for (int i = 0; i < 66; i++) { - if (openSeeData[0].got3DPoints && openSeeData[0].confidence[i] > minConfidence) { + if (openSeeData.got3DPoints && openSeeData.confidence[i] > minConfidence) { Renderer renderer = gameObjects[i].GetComponent(); - renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData[0].confidence[i])); - Vector3 pt = openSeeData[0].points3D[i]; + renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i])); + Vector3 pt = openSeeData.points3D[i]; pt.x = -pt.x; gameObjects[i].transform.localPosition = pt; } else { @@ -59,14 +63,14 @@ public class OpenSeeShowPoints : MonoBehaviour { } } if (applyTranslation) { - Vector3 v = openSeeData[0].translation; + Vector3 v = openSeeData.translation; v.x = -v.x; v.z = -v.z; transform.localPosition = v; } if (applyRotation) { Quaternion offset = Quaternion.Euler(0f, 0f, -90f); - Quaternion convertedQuaternion = new Quaternion(-openSeeData[0].rawQuaternion.y, -openSeeData[0].rawQuaternion.x, openSeeData[0].rawQuaternion.z, openSeeData[0].rawQuaternion.w) * offset; + Quaternion convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w) * offset; transform.localRotation = convertedQuaternion; } } else { @@ -77,8 +81,8 @@ public class OpenSeeShowPoints : MonoBehaviour { float maxX = -1.0f; float maxY = -1.0f; for (int i = 0; i < 66; i++) { - float x = openSeeData[0].points[i].x; - float y = -openSeeData[0].points[i].y; + float x = openSeeData.points[i].x; + float y = -openSeeData.points[i].y; if (minX > x) { minX = x; } @@ -102,9 +106,9 @@ public class OpenSeeShowPoints : MonoBehaviour { for (int i = 0; i < 66; i++) { Renderer renderer = gameObjects[i].GetComponent(); - renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData[0].confidence[i])); - float x = openSeeData[0].points[i].x; - float y = -openSeeData[0].points[i].y; + renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i])); + float x = openSeeData.points[i].x; + float y = -openSeeData.points[i].y; Vector3 position = new Vector3(x, y, 0.0f); position = position - new Vector3(minX, minY, 0.0f); position.x = position.x / (maxX - minX); diff --git a/facetracker.py b/facetracker.py index 1be3bf0..a98415c 100644 --- a/facetracker.py +++ b/facetracker.py @@ -1,3 +1,4 @@ +import copy import os import sys import argparse @@ -29,6 +30,7 @@ parser.add_argument("--model", type=int, help="This can be used to select the tr parser.add_argument("--model-dir", help="This can be used to specify the path to the directory containing the .onnx model files", default=None) parser.add_argument("--high-quality-3d", type=int, help="When set to 1, more nose points are used when estimating the face pose", default=1) parser.add_argument("--gaze-tracking", type=int, help="When set to 1, experimental blink detection and gaze tracking are enabled, which makes things slightly slower", default=1) +parser.add_argument("--face-id-offset", type=int, help="When set, this offset is added to all face ids, which can be useful for mixing tracking data from multiple network sources", default=0) args = parser.parse_args() os.environ["OMP_NUM_THREADS"] = "1" @@ -116,6 +118,8 @@ try: packet = bytearray() detected = False for face_num, f in enumerate(faces): + f = copy.copy(f) + f.id += args.face_id_offset right_state = "O" if f.eye_blink[0] > 0.5 else "-" left_state = "O" if f.eye_blink[1] > 0.5 else "-" if args.silent == 0: