copilot/OpenSeeFace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

The face tracker can now offset face ids by a given value. This makes it possible to easily mix tracking input from multiple network sources.

Browse source
This commit is contained in:
Emiliana 2020-02-16 15:58:47 +01:00
parent c2eb544096
commit 633293eb52
6 changed files with 64 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

29
Unity/OpenSee.cs
View file

@ -139,11 +139,20 @@ public class OpenSee : MonoBehaviour {
} }
} }
private Dictionary<int, OpenSeeData> openSeeDataMap;
private Socket socket; private Socket socket;
private byte[] buffer; private byte[] buffer;
private Thread receiveThread = null; private Thread receiveThread = null;
private volatile bool stopReception = false; private volatile bool stopReception = false;
public OpenSeeData GetOpenSeeData(int faceId) {
if (openSeeDataMap == null)
return null;
if (!openSeeDataMap.ContainsKey(faceId))
return null;
return openSeeDataMap[faceId];
}
void performReception() { void performReception() {
EndPoint senderRemote = new IPEndPoint(IPAddress.Any, 0); EndPoint senderRemote = new IPEndPoint(IPAddress.Any, 0);
while (!stopReception) { while (!stopReception) {
@ -154,18 +163,20 @@ public class OpenSee : MonoBehaviour {
} }
receivedPackets++; receivedPackets++;
int i = 0; int i = 0;
OpenSeeData[] newData = new OpenSeeData[receivedBytes / packetFrameSize];
for (int offset = 0; offset < receivedBytes; offset += packetFrameSize) { for (int offset = 0; offset < receivedBytes; offset += packetFrameSize) {
newData[i] = new OpenSeeData(); OpenSeeData newData = new OpenSeeData();
newData[i].readFromPacket(buffer, offset); newData.readFromPacket(buffer, offset);
openSeeDataMap[newData.id] = newData;
i++; i++;
} }
trackingData = newData; trackingData = new OpenSeeData[openSeeDataMap.Count];
openSeeDataMap.Values.CopyTo(trackingData, 0);
} catch { } } catch { }
} }
} }
void Start () { void Start () {
openSeeDataMap = new Dictionary<int, OpenSeeData>();
buffer = new byte[65535]; buffer = new byte[65535];
socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
@ -181,7 +192,7 @@ public class OpenSee : MonoBehaviour {
void Update () { void Update () {
} }
void OnApplicationQuit() { void EndReceiver() {
if (receiveThread != null) { if (receiveThread != null) {
stopReception = true; stopReception = true;
receiveThread.Join(); receiveThread.Join();
@ -190,6 +201,14 @@ public class OpenSee : MonoBehaviour {
if (socket != null) if (socket != null)
socket.Close(); socket.Close();
} }
void OnApplicationQuit() {
EndReceiver();
}
void OnDestroy() {
EndReceiver();
}
} }
} }

36
Unity/OpenSeeExpression.cs
View file

@ -137,6 +137,7 @@ public class OpenSeeExpression : MonoBehaviour
} }
} }
private OpenSee.OpenSeeData openSeeData = null;
private Dictionary<string, List<float[]>> expressions; private Dictionary<string, List<float[]>> expressions;
private SVMModel model = null; private SVMModel model = null;
private string[] classLabels = null; private string[] classLabels = null;
@ -394,16 +395,10 @@ public class OpenSeeExpression : MonoBehaviour
ResetInfo(); ResetInfo();
return false; return false;
} }
OpenSee.OpenSeeData[] openSeeData = openSee.trackingData; openSeeData = openSee.GetOpenSeeData(faceId);
if (openSeeData == null) if (openSeeData == null || openSeeData.time <= lastCapture)
return false; return false;
OpenSee.OpenSeeData t = null; float[] faceData = GetData(openSeeData);
foreach (OpenSee.OpenSeeData data in openSeeData)
if (data.id == faceId)
t = data;
if (t == null || t.time <= lastCapture)
return false;
float[] faceData = GetData(t);
float[] predictionData = new float[cols]; float[] predictionData = new float[cols];
for (int i = 0; i < cols; i++) for (int i = 0; i < cols; i++)
predictionData[i] = faceData[indices[i]]; predictionData[i] = faceData[indices[i]];
@ -436,14 +431,8 @@ public class OpenSeeExpression : MonoBehaviour
if (calibrationExpression == "") if (calibrationExpression == "")
return false; return false;
if (expressions.ContainsKey(calibrationExpression) || expressions.Keys.Count < 10) { if (expressions.ContainsKey(calibrationExpression) || expressions.Keys.Count < 10) {
OpenSee.OpenSeeData[] openSeeData = openSee.trackingData; openSeeData = openSee.GetOpenSeeData(faceId);
if (openSeeData == null) if (openSeeData == null || openSeeData.time <= lastCapture)
return false;
OpenSee.OpenSeeData t = null;
foreach (OpenSee.OpenSeeData data in openSeeData)
if (data.id == faceId)
t = data;
if (t == null || t.time <= lastCapture)
return false; return false;
if (!expressions.ContainsKey(calibrationExpression)) { if (!expressions.ContainsKey(calibrationExpression)) {
expressions.Add(calibrationExpression, new List<float[]>()); expressions.Add(calibrationExpression, new List<float[]>());
@ -461,9 +450,9 @@ public class OpenSeeExpression : MonoBehaviour
if (frameCount % skip != 0) if (frameCount % skip != 0)
return true; return true;
if (percentRecorded >= 100f) { if (percentRecorded >= 100f) {
list[rnd.Next(maxSamples)] = GetData(t); list[rnd.Next(maxSamples)] = GetData(openSeeData);
} else } else
list.Add(GetData(t)); list.Add(GetData(openSeeData));
percentRecorded = 100f * (float)list.Count/(float)maxSamples; percentRecorded = 100f * (float)list.Count/(float)maxSamples;
return true; return true;
} else { } else {
@ -566,16 +555,9 @@ public class OpenSeeExpression : MonoBehaviour
else else
percentRecorded = 0f; percentRecorded = 0f;
OpenSee.OpenSeeData[] openSeeData = openSee.trackingData;
if (openSeeData == null) if (openSeeData == null)
return; return;
OpenSee.OpenSeeData t = null; lastCapture = openSeeData.time;
foreach (OpenSee.OpenSeeData data in openSeeData)
if (data.id == faceId)
t = data;
if (t == null || t.time <= lastCapture)
return;
lastCapture = t.time;
} }
} }

21
Unity/OpenSeeIKTarget.cs
View file

@ -61,27 +61,18 @@ public class OpenSeeIKTarget : MonoBehaviour
} }
void RunUpdate() { void RunUpdate() {
var openSeeData = openSee.trackingData; var openSeeData = openSee.GetOpenSeeData(faceId);
if (openSeeData == null || openSeeData.Length < 1) if (openSeeData == null)
return; return;
int idx = -1; if (openSeeData.time > updated) {
int i = 0; updated = openSeeData.time;
foreach (var item in openSeeData) {
if (item.id == faceId)
idx = i;
i++;
}
if (idx < 0)
return;
if (openSeeData[idx].time > updated) {
updated = openSeeData[idx].time;
} else { } else {
Interpolate(); Interpolate();
return; return;
} }
Quaternion convertedQuaternion = new Quaternion(-openSeeData[idx].rawQuaternion.y, -openSeeData[idx].rawQuaternion.x, openSeeData[idx].rawQuaternion.z, openSeeData[idx].rawQuaternion.w); Quaternion convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w);
Vector3 t = openSeeData[idx].translation; Vector3 t = openSeeData.translation;
t.x = -t.x; t.x = -t.x;
t.z = -t.z; t.z = -t.z;

2
Unity/OpenSeeLauncher.cs
View file

@ -29,7 +29,7 @@ public class OpenSeeLauncher : MonoBehaviour {
[Tooltip("The path, where the .onnx model files can be found")] [Tooltip("The path, where the .onnx model files can be found")]
public string modelPath = "models\\"; public string modelPath = "models\\";
[Tooltip("Additional options that should be passed to the face tracker.")] [Tooltip("Additional options that should be passed to the face tracker.")]
public List<string> commandlineOptions = new List<string>(new string[] { "--silent", "1", "--max-threads", "2", "--model", "3" }); public List<string> commandlineOptions = new List<string>(new string[] { "--silent", "1", "--max-threads", "4" });
[Header("Runtime information")] [Header("Runtime information")]
[Tooltip("This field shows if the tracker is currently alive.")] [Tooltip("This field shows if the tracker is currently alive.")]
public bool trackerAlive = false; public bool trackerAlive = false;

32
Unity/OpenSeeShowPoints.cs
View file

@ -6,12 +6,13 @@ namespace OpenSee {
public class OpenSeeShowPoints : MonoBehaviour { public class OpenSeeShowPoints : MonoBehaviour {
public OpenSee openSee = null; public OpenSee openSee = null;
public int faceId = 0;
public bool show3DPoints = true; public bool show3DPoints = true;
public bool applyTranslation = false; public bool applyTranslation = false;
public bool applyRotation = false; public bool applyRotation = false;
[Range(0, 1)] [Range(0, 1)]
public float minConfidence = 0.20f; public float minConfidence = 0.20f;
private OpenSee.OpenSeeData[] openSeeData; private OpenSee.OpenSeeData openSeeData;
private GameObject[] gameObjects; private GameObject[] gameObjects;
private GameObject centerBall; private GameObject centerBall;
private double updated = 0.0; private double updated = 0.0;
@ -36,21 +37,24 @@ public class OpenSeeShowPoints : MonoBehaviour {
void Update () { void Update () {
if (!openSee) if (!openSee)
return; return;
openSeeData = openSee.trackingData; /*openSeeData = openSee.trackingData;
if (openSeeData == null || openSeeData.Length < 1) if (openSeeData == null || openSeeData.Length < 1)
return;*/
openSeeData = openSee.GetOpenSeeData(faceId);
if (openSeeData == null)
return; return;
if (openSeeData[0].time > updated) { if (openSeeData.time > updated) {
updated = openSeeData[0].time; updated = openSeeData.time;
} else { } else {
return; return;
} }
if (show3DPoints) { if (show3DPoints) {
centerBall.gameObject.SetActive(false); centerBall.gameObject.SetActive(false);
for (int i = 0; i < 66; i++) { for (int i = 0; i < 66; i++) {
if (openSeeData[0].got3DPoints && openSeeData[0].confidence[i] > minConfidence) { if (openSeeData.got3DPoints && openSeeData.confidence[i] > minConfidence) {
Renderer renderer = gameObjects[i].GetComponent<Renderer>(); Renderer renderer = gameObjects[i].GetComponent<Renderer>();
renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData[0].confidence[i])); renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i]));
Vector3 pt = openSeeData[0].points3D[i]; Vector3 pt = openSeeData.points3D[i];
pt.x = -pt.x; pt.x = -pt.x;
gameObjects[i].transform.localPosition = pt; gameObjects[i].transform.localPosition = pt;
} else { } else {
@ -59,14 +63,14 @@ public class OpenSeeShowPoints : MonoBehaviour {
} }
} }
if (applyTranslation) { if (applyTranslation) {
Vector3 v = openSeeData[0].translation; Vector3 v = openSeeData.translation;
v.x = -v.x; v.x = -v.x;
v.z = -v.z; v.z = -v.z;
transform.localPosition = v; transform.localPosition = v;
} }
if (applyRotation) { if (applyRotation) {
Quaternion offset = Quaternion.Euler(0f, 0f, -90f); Quaternion offset = Quaternion.Euler(0f, 0f, -90f);
Quaternion convertedQuaternion = new Quaternion(-openSeeData[0].rawQuaternion.y, -openSeeData[0].rawQuaternion.x, openSeeData[0].rawQuaternion.z, openSeeData[0].rawQuaternion.w) * offset; Quaternion convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w) * offset;
transform.localRotation = convertedQuaternion; transform.localRotation = convertedQuaternion;
} }
} else { } else {
@ -77,8 +81,8 @@ public class OpenSeeShowPoints : MonoBehaviour {
float maxX = -1.0f; float maxX = -1.0f;
float maxY = -1.0f; float maxY = -1.0f;
for (int i = 0; i < 66; i++) { for (int i = 0; i < 66; i++) {
float x = openSeeData[0].points[i].x; float x = openSeeData.points[i].x;
float y = -openSeeData[0].points[i].y; float y = -openSeeData.points[i].y;
if (minX > x) { if (minX > x) {
minX = x; minX = x;
} }
@ -102,9 +106,9 @@ public class OpenSeeShowPoints : MonoBehaviour {
for (int i = 0; i < 66; i++) { for (int i = 0; i < 66; i++) {
Renderer renderer = gameObjects[i].GetComponent<Renderer>(); Renderer renderer = gameObjects[i].GetComponent<Renderer>();
renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData[0].confidence[i])); renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i]));
float x = openSeeData[0].points[i].x; float x = openSeeData.points[i].x;
float y = -openSeeData[0].points[i].y; float y = -openSeeData.points[i].y;
Vector3 position = new Vector3(x, y, 0.0f); Vector3 position = new Vector3(x, y, 0.0f);
position = position - new Vector3(minX, minY, 0.0f); position = position - new Vector3(minX, minY, 0.0f);
position.x = position.x / (maxX - minX); position.x = position.x / (maxX - minX);

4
facetracker.py
View file

@ -1,3 +1,4 @@
import copy
import os import os
import sys import sys
import argparse import argparse
@ -29,6 +30,7 @@ parser.add_argument("--model", type=int, help="This can be used to select the tr
parser.add_argument("--model-dir", help="This can be used to specify the path to the directory containing the .onnx model files", default=None) parser.add_argument("--model-dir", help="This can be used to specify the path to the directory containing the .onnx model files", default=None)
parser.add_argument("--high-quality-3d", type=int, help="When set to 1, more nose points are used when estimating the face pose", default=1) parser.add_argument("--high-quality-3d", type=int, help="When set to 1, more nose points are used when estimating the face pose", default=1)
parser.add_argument("--gaze-tracking", type=int, help="When set to 1, experimental blink detection and gaze tracking are enabled, which makes things slightly slower", default=1) parser.add_argument("--gaze-tracking", type=int, help="When set to 1, experimental blink detection and gaze tracking are enabled, which makes things slightly slower", default=1)
parser.add_argument("--face-id-offset", type=int, help="When set, this offset is added to all face ids, which can be useful for mixing tracking data from multiple network sources", default=0)
args = parser.parse_args() args = parser.parse_args()
os.environ["OMP_NUM_THREADS"] = "1" os.environ["OMP_NUM_THREADS"] = "1"
@ -116,6 +118,8 @@ try:
packet = bytearray() packet = bytearray()
detected = False detected = False
for face_num, f in enumerate(faces): for face_num, f in enumerate(faces):
f = copy.copy(f)
f.id += args.face_id_offset
right_state = "O" if f.eye_blink[0] > 0.5 else "-" right_state = "O" if f.eye_blink[0] > 0.5 else "-"
left_state = "O" if f.eye_blink[1] > 0.5 else "-" left_state = "O" if f.eye_blink[1] > 0.5 else "-"
if args.silent == 0: if args.silent == 0: