copilot/OpenSeeFace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

The face tracker can now offset face ids by a given value. This makes it possible to easily mix tracking input from multiple network sources.

Browse source
This commit is contained in:
Emiliana 2020-02-16 15:58:47 +01:00
parent c2eb544096
commit 633293eb52
6 changed files with 64 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

29
Unity/OpenSee.cs
View file

@ -139,11 +139,20 @@ public class OpenSee : MonoBehaviour {
}
}
private Dictionary<int, OpenSeeData> openSeeDataMap;
private Socket socket;
private byte[] buffer;
private Thread receiveThread = null;
private volatile bool stopReception = false;
public OpenSeeData GetOpenSeeData(int faceId) {
if (openSeeDataMap == null)
return null;
if (!openSeeDataMap.ContainsKey(faceId))
return null;
return openSeeDataMap[faceId];
}
void performReception() {
EndPoint senderRemote = new IPEndPoint(IPAddress.Any, 0);
while (!stopReception) {
@ -154,18 +163,20 @@ public class OpenSee : MonoBehaviour {
}
receivedPackets++;
int i = 0;
OpenSeeData[] newData = new OpenSeeData[receivedBytes / packetFrameSize];
for (int offset = 0; offset < receivedBytes; offset += packetFrameSize) {
newData[i] = new OpenSeeData();
newData[i].readFromPacket(buffer, offset);
OpenSeeData newData = new OpenSeeData();
newData.readFromPacket(buffer, offset);
openSeeDataMap[newData.id] = newData;
i++;
}
trackingData = newData;
trackingData = new OpenSeeData[openSeeDataMap.Count];
openSeeDataMap.Values.CopyTo(trackingData, 0);
} catch { }
}
}
void Start () {
openSeeDataMap = new Dictionary<int, OpenSeeData>();
buffer = new byte[65535];
socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
@ -181,7 +192,7 @@ public class OpenSee : MonoBehaviour {
void Update () {
}
void OnApplicationQuit() {
void EndReceiver() {
if (receiveThread != null) {
stopReception = true;
receiveThread.Join();
@ -190,6 +201,14 @@ public class OpenSee : MonoBehaviour {
if (socket != null)
socket.Close();
}
void OnApplicationQuit() {
EndReceiver();
}
void OnDestroy() {
EndReceiver();
}
}
}

36
Unity/OpenSeeExpression.cs
View file

@ -137,6 +137,7 @@ public class OpenSeeExpression : MonoBehaviour
}
}
private OpenSee.OpenSeeData openSeeData = null;
private Dictionary<string, List<float[]>> expressions;
private SVMModel model = null;
private string[] classLabels = null;
@ -394,16 +395,10 @@ public class OpenSeeExpression : MonoBehaviour
ResetInfo();
return false;
}
OpenSee.OpenSeeData[] openSeeData = openSee.trackingData;
if (openSeeData == null)
openSeeData = openSee.GetOpenSeeData(faceId);
if (openSeeData == null || openSeeData.time <= lastCapture)
return false;
OpenSee.OpenSeeData t = null;
foreach (OpenSee.OpenSeeData data in openSeeData)
if (data.id == faceId)
t = data;
if (t == null || t.time <= lastCapture)
return false;
float[] faceData = GetData(t);
float[] faceData = GetData(openSeeData);
float[] predictionData = new float[cols];
for (int i = 0; i < cols; i++)
predictionData[i] = faceData[indices[i]];
@ -436,14 +431,8 @@ public class OpenSeeExpression : MonoBehaviour
if (calibrationExpression == "")
return false;
if (expressions.ContainsKey(calibrationExpression) || expressions.Keys.Count < 10) {
OpenSee.OpenSeeData[] openSeeData = openSee.trackingData;
if (openSeeData == null)
return false;
OpenSee.OpenSeeData t = null;
foreach (OpenSee.OpenSeeData data in openSeeData)
if (data.id == faceId)
t = data;
if (t == null || t.time <= lastCapture)
openSeeData = openSee.GetOpenSeeData(faceId);
if (openSeeData == null || openSeeData.time <= lastCapture)
return false;
if (!expressions.ContainsKey(calibrationExpression)) {
expressions.Add(calibrationExpression, new List<float[]>());
@ -461,9 +450,9 @@ public class OpenSeeExpression : MonoBehaviour
if (frameCount % skip != 0)
return true;
if (percentRecorded >= 100f) {
list[rnd.Next(maxSamples)] = GetData(t);
list[rnd.Next(maxSamples)] = GetData(openSeeData);
} else
list.Add(GetData(t));
list.Add(GetData(openSeeData));
percentRecorded = 100f * (float)list.Count/(float)maxSamples;
return true;
} else {
@ -566,16 +555,9 @@ public class OpenSeeExpression : MonoBehaviour
else
percentRecorded = 0f;
OpenSee.OpenSeeData[] openSeeData = openSee.trackingData;
if (openSeeData == null)
return;
OpenSee.OpenSeeData t = null;
foreach (OpenSee.OpenSeeData data in openSeeData)
if (data.id == faceId)
t = data;
if (t == null || t.time <= lastCapture)
return;
lastCapture = t.time;
lastCapture = openSeeData.time;
}
}

21
Unity/OpenSeeIKTarget.cs
View file

@ -61,27 +61,18 @@ public class OpenSeeIKTarget : MonoBehaviour
}
void RunUpdate() {
var openSeeData = openSee.trackingData;
if (openSeeData == null || openSeeData.Length < 1)
var openSeeData = openSee.GetOpenSeeData(faceId);
if (openSeeData == null)
return;
int idx = -1;
int i = 0;
foreach (var item in openSeeData) {
if (item.id == faceId)
idx = i;
i++;
}
if (idx < 0)
return;
if (openSeeData[idx].time > updated) {
updated = openSeeData[idx].time;
if (openSeeData.time > updated) {
updated = openSeeData.time;
} else {
Interpolate();
return;
}
Quaternion convertedQuaternion = new Quaternion(-openSeeData[idx].rawQuaternion.y, -openSeeData[idx].rawQuaternion.x, openSeeData[idx].rawQuaternion.z, openSeeData[idx].rawQuaternion.w);
Vector3 t = openSeeData[idx].translation;
Quaternion convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w);
Vector3 t = openSeeData.translation;
t.x = -t.x;
t.z = -t.z;

2
Unity/OpenSeeLauncher.cs
View file

@ -29,7 +29,7 @@ public class OpenSeeLauncher : MonoBehaviour {
[Tooltip("The path, where the .onnx model files can be found")]
public string modelPath = "models\\";
[Tooltip("Additional options that should be passed to the face tracker.")]
public List<string> commandlineOptions = new List<string>(new string[] { "--silent", "1", "--max-threads", "2", "--model", "3" });
public List<string> commandlineOptions = new List<string>(new string[] { "--silent", "1", "--max-threads", "4" });
[Header("Runtime information")]
[Tooltip("This field shows if the tracker is currently alive.")]
public bool trackerAlive = false;

32
Unity/OpenSeeShowPoints.cs
View file

@ -6,12 +6,13 @@ namespace OpenSee {
public class OpenSeeShowPoints : MonoBehaviour {
public OpenSee openSee = null;
public int faceId = 0;
public bool show3DPoints = true;
public bool applyTranslation = false;
public bool applyRotation = false;
[Range(0, 1)]
public float minConfidence = 0.20f;
private OpenSee.OpenSeeData[] openSeeData;
private OpenSee.OpenSeeData openSeeData;
private GameObject[] gameObjects;
private GameObject centerBall;
private double updated = 0.0;
@ -36,21 +37,24 @@ public class OpenSeeShowPoints : MonoBehaviour {
void Update () {
if (!openSee)
return;
openSeeData = openSee.trackingData;
/*openSeeData = openSee.trackingData;
if (openSeeData == null || openSeeData.Length < 1)
return;*/
openSeeData = openSee.GetOpenSeeData(faceId);
if (openSeeData == null)
return;
if (openSeeData[0].time > updated) {
updated = openSeeData[0].time;
if (openSeeData.time > updated) {
updated = openSeeData.time;
} else {
return;
}
if (show3DPoints) {
centerBall.gameObject.SetActive(false);
for (int i = 0; i < 66; i++) {
if (openSeeData[0].got3DPoints && openSeeData[0].confidence[i] > minConfidence) {
if (openSeeData.got3DPoints && openSeeData.confidence[i] > minConfidence) {
Renderer renderer = gameObjects[i].GetComponent<Renderer>();
renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData[0].confidence[i]));
Vector3 pt = openSeeData[0].points3D[i];
renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i]));
Vector3 pt = openSeeData.points3D[i];
pt.x = -pt.x;
gameObjects[i].transform.localPosition = pt;
} else {
@ -59,14 +63,14 @@ public class OpenSeeShowPoints : MonoBehaviour {
}
}
if (applyTranslation) {
Vector3 v = openSeeData[0].translation;
Vector3 v = openSeeData.translation;
v.x = -v.x;
v.z = -v.z;
transform.localPosition = v;
}
if (applyRotation) {
Quaternion offset = Quaternion.Euler(0f, 0f, -90f);
Quaternion convertedQuaternion = new Quaternion(-openSeeData[0].rawQuaternion.y, -openSeeData[0].rawQuaternion.x, openSeeData[0].rawQuaternion.z, openSeeData[0].rawQuaternion.w) * offset;
Quaternion convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w) * offset;
transform.localRotation = convertedQuaternion;
}
} else {
@ -77,8 +81,8 @@ public class OpenSeeShowPoints : MonoBehaviour {
float maxX = -1.0f;
float maxY = -1.0f;
for (int i = 0; i < 66; i++) {
float x = openSeeData[0].points[i].x;
float y = -openSeeData[0].points[i].y;
float x = openSeeData.points[i].x;
float y = -openSeeData.points[i].y;
if (minX > x) {
minX = x;
}
@ -102,9 +106,9 @@ public class OpenSeeShowPoints : MonoBehaviour {
for (int i = 0; i < 66; i++) {
Renderer renderer = gameObjects[i].GetComponent<Renderer>();
renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData[0].confidence[i]));
float x = openSeeData[0].points[i].x;
float y = -openSeeData[0].points[i].y;
renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i]));
float x = openSeeData.points[i].x;
float y = -openSeeData.points[i].y;
Vector3 position = new Vector3(x, y, 0.0f);
position = position - new Vector3(minX, minY, 0.0f);
position.x = position.x / (maxX - minX);

4
facetracker.py
View file

@ -1,3 +1,4 @@
import copy
import os
import sys
import argparse
@ -29,6 +30,7 @@ parser.add_argument("--model", type=int, help="This can be used to select the tr
parser.add_argument("--model-dir", help="This can be used to specify the path to the directory containing the .onnx model files", default=None)
parser.add_argument("--high-quality-3d", type=int, help="When set to 1, more nose points are used when estimating the face pose", default=1)
parser.add_argument("--gaze-tracking", type=int, help="When set to 1, experimental blink detection and gaze tracking are enabled, which makes things slightly slower", default=1)
parser.add_argument("--face-id-offset", type=int, help="When set, this offset is added to all face ids, which can be useful for mixing tracking data from multiple network sources", default=0)
args = parser.parse_args()
os.environ["OMP_NUM_THREADS"] = "1"
@ -116,6 +118,8 @@ try:
packet = bytearray()
detected = False
for face_num, f in enumerate(faces):
f = copy.copy(f)
f.id += args.face_id_offset
right_state = "O" if f.eye_blink[0] > 0.5 else "-"
left_state = "O" if f.eye_blink[1] > 0.5 else "-"
if args.silent == 0: