copilot/OpenSeeFace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

This adds manually designed features to the OpenSeeData.

Browse source
This commit is contained in:
Emiliana 2020-03-01 14:23:56 +01:00
parent 7706372a7a
commit 04d7f5b8b3
7 changed files with 421 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

52
Unity/OpenSee.cs
View file

@ -16,7 +16,7 @@ public class OpenSee : MonoBehaviour {
public int listenPort = 11573;
private const int nPoints = 68;
private const int packetFrameSize = 8 + 4 + 2 * 4 + 2 * 4 + 1 + 4 + 3 * 4 + 3 * 4 + 4 * 4 + 4 * 68 + 4 * 2 * 68 + 4 * 3 * 70;
private const int packetFrameSize = 8 + 4 + 2 * 4 + 2 * 4 + 1 + 4 + 3 * 4 + 3 * 4 + 4 * 4 + 4 * 68 + 4 * 2 * 68 + 4 * 3 * 70 + 4 * 14;
[Header("Tracking data")]
[Tooltip("This is an informational property that tells you how many packets have been received")]
@ -58,6 +58,40 @@ public class OpenSee : MonoBehaviour {
public Vector2[] points;
[Tooltip("These are 3D points estimated from the 2D points. The should be rotation and translation compensated. There are 70 points with guesses for the eyeball center positions being added at the end of the 68 2D points.")]
public Vector3[] points3D;
[Tooltip("This field contains a number of action unit like features.")]
public OpenSeeFeatures features;
[System.Serializable]
public class OpenSeeFeatures {
[Tooltip("This field indicates whether the left eye is opened(0) or closed (-1). A value of 1 means open wider than normal.")]
public float EyeLeft;
[Tooltip("This field indicates whether the right eye is opened(0) or closed (-1). A value of 1 means open wider than normal.")]
public float EyeRight;
[Tooltip("This field indicates how steep the left eyebrow is, compared to the median steepness.")]
public float EyebrowSteepnessLeft;
[Tooltip("This field indicates how far up or down the left eyebrow is, compared to its median position.")]
public float EyebrowUpDownLeft;
[Tooltip("This field indicates how quirked the left eyebrow is, compared to its median quirk.")]
public float EyebrowQuirkLeft;
[Tooltip("This field indicates how steep the right eyebrow is, compared to the average steepness.")]
public float EyebrowSteepnessRight;
[Tooltip("This field indicates how far up or down the right eyebrow is, compared to its median position.")]
public float EyebrowUpDownRight;
[Tooltip("This field indicates how quirked the right eyebrow is, compared to its median quirk.")]
public float EyebrowQuirkRight;
[Tooltip("This field indicates how far up or down the left mouth corner is, compared to its median position.")]
public float MouthCornerUpDownLeft;
[Tooltip("This field indicates how far in or out the left mouth corner is, compared to its median position.")]
public float MouthCornerInOutLeft;
[Tooltip("This field indicates how far up or down the right mouth corner is, compared to its median position.")]
public float MouthCornerUpDownRight;
[Tooltip("This field indicates how far in or out the right mouth corner is, compared to its median position.")]
public float MouthCornerInOutRight;
[Tooltip("This field indicates how open or closed the mouth is, compared to its median pose.")]
public float MouthOpen;
[Tooltip("This field indicates how wide the mouth is, compared to its median pose.")]
public float MouthWide;
}
public OpenSeeData() {
confidence = new float[nPoints];
@ -139,6 +173,22 @@ public class OpenSee : MonoBehaviour {
rightGaze = Quaternion.LookRotation(swapX(points3D[66]) - swapX(points3D[68])) * Quaternion.AngleAxis(180, Vector3.right) * Quaternion.AngleAxis(180, Vector3.forward);
leftGaze = Quaternion.LookRotation(swapX(points3D[67]) - swapX(points3D[69])) * Quaternion.AngleAxis(180, Vector3.right) * Quaternion.AngleAxis(180, Vector3.forward);
features = new OpenSeeFeatures();
features.EyeLeft = readFloat(b, ref o);
features.EyeRight = readFloat(b, ref o);
features.EyebrowSteepnessLeft = readFloat(b, ref o);
features.EyebrowUpDownLeft = readFloat(b, ref o);
features.EyebrowQuirkLeft = readFloat(b, ref o);
features.EyebrowSteepnessRight = readFloat(b, ref o);
features.EyebrowUpDownRight = readFloat(b, ref o);
features.EyebrowQuirkRight = readFloat(b, ref o);
features.MouthCornerUpDownLeft = readFloat(b, ref o);
features.MouthCornerInOutLeft = readFloat(b, ref o);
features.MouthCornerUpDownRight = readFloat(b, ref o);
features.MouthCornerInOutRight = readFloat(b, ref o);
features.MouthOpen = readFloat(b, ref o);
features.MouthWide = readFloat(b, ref o);
}
}

65
Unity/OpenSeeExpression.cs
View file

@ -27,6 +27,8 @@ public class OpenSeeExpression : MonoBehaviour
public string filename = "";
[Tooltip("When enabled, training will be done in such a way that the model can output probabilities. This is a lot slower!")]
public bool enableProbabilityTraining = false;
[Tooltip("This is the SVM training C value.")]
public float C = 2f;
[Header("Toggles")]
[Tooltip("When enabled, calibration data will be collected from the given OpenSee component.")]
public bool recording = false;
@ -88,6 +90,8 @@ public class OpenSeeExpression : MonoBehaviour
public bool pointsLipUpper = true;
[Tooltip("When enabled, the points of the lower lip will be used to determine the expression.")]
public bool pointsLipLower = true;
[Tooltip("When enabled, certain manually designed features will be used to determine the expression.")]
public bool features = true;
}
[Serializable]
private class OpenSeeExpressionRepresentation {
@ -96,6 +100,7 @@ public class OpenSeeExpression : MonoBehaviour
private string[] classLabels = null;
private int[] indices = null;
private PointSelection pointSelection;
private bool newModel = false;
static public void LoadSerialized(byte[] modelBytes, out Dictionary<string, List<float[]>> expressions, out SVMModel model, out string[] classLabels, out int[] indices, out PointSelection pointSelection) {
IFormatter formatter = new BinaryFormatter();
@ -117,6 +122,8 @@ public class OpenSeeExpression : MonoBehaviour
}
if (pointSelection == null)
pointSelection = new PointSelection();
if (!oser.newModel)
pointSelection.features = true;
}
static public byte[] ToSerialized(Dictionary<string, List<float[]>> expressions, SVMModel model, string[] classLabels, int[] indices, PointSelection pointSelection) {
@ -126,6 +133,7 @@ public class OpenSeeExpression : MonoBehaviour
oser.classLabels = classLabels;
oser.indices = indices;
oser.pointSelection = pointSelection;
oser.newModel = true;
IFormatter formatter = new BinaryFormatter();
MemoryStream memoryStream = new MemoryStream();
@ -141,9 +149,9 @@ public class OpenSeeExpression : MonoBehaviour
private Dictionary<string, List<float[]>> expressions;
private SVMModel model = null;
private string[] classLabels = null;
private int maxSamples = 1000; // Allows 10 expressions without exceeding 10000 rows in total for SVM training, which is a commonly seen upper limit.
// rightEyeOpen, leftEyeOpen, translation, rawQuaternion, rawEuler, confidence, points/(width, height), points3D
private int colsFull = 1 + 1 + 3 + 4 + 3 + /*66 + 2 * 66 +*/ 3 * 66;
private int maxSamples = 1666;
// rightEyeOpen, leftEyeOpen, translation, rawQuaternion, rawEuler, confidence, points/(width, height), points3D, features
private int colsFull = 1 + 1 + 3 + 4 + 3 + /*66 + 2 * 66 +*/ 3 * 66 + 14;
private int colsBase = 1 + 1 + 3 + 4 + 3;
private int cols;
private double lastCapture = 0.0;
@ -235,6 +243,11 @@ public class OpenSeeExpression : MonoBehaviour
indexList.Add(colsBase + i * 3 + 2);
cols += 3;
}
if (pointSelection.features)
for (int i = 0; i < 14; i++) {
indexList.Add(colsBase + 66 * 3 + i);
cols++;
}
indices = indexList.ToArray();
}
@ -294,10 +307,26 @@ public class OpenSeeExpression : MonoBehaviour
data[12 + /*66 + 2 * 66 +*/ 3 * i + 1] = t.points3D[i].y;
data[12 + /*66 + 2 * 66 +*/ 3 * i + 2] = t.points3D[i].z;
}
data[12 + 66 * 3 + 0] = t.features.EyeLeft;
data[12 + 66 * 3 + 1] = t.features.EyeRight;
data[12 + 66 * 3 + 2] = t.features.EyebrowSteepnessLeft;
data[12 + 66 * 3 + 3] = t.features.EyebrowUpDownLeft;
data[12 + 66 * 3 + 4] = t.features.EyebrowQuirkLeft;
data[12 + 66 * 3 + 5] = t.features.EyebrowSteepnessRight;
data[12 + 66 * 3 + 6] = t.features.EyebrowUpDownRight;
data[12 + 66 * 3 + 7] = t.features.EyebrowQuirkRight;
data[12 + 66 * 3 + 8] = t.features.MouthCornerUpDownLeft;
data[12 + 66 * 3 + 9] = t.features.MouthCornerInOutLeft;
data[12 + 66 * 3 + 10] = t.features.MouthCornerUpDownRight;
data[12 + 66 * 3 + 11] = t.features.MouthCornerInOutRight;
data[12 + 66 * 3 + 12] = t.features.MouthOpen;
data[12 + 66 * 3 + 13] = t.features.MouthWide;
return data;
}
public bool TrainModel() {
Debug.Log("Training model");
Debug.Log("---------------------");
train = false;
if (openSee == null) {
ResetInfo();
@ -310,7 +339,8 @@ public class OpenSeeExpression : MonoBehaviour
List<string> accuracyWarnings = new List<string>();
int samples = 0;
foreach (string key in expressions.Keys) {
List<float[]> list = expressions[key];
List<float[]> list = new List<float[]>(expressions[key]);
list.RemoveAll(x => x.Length != colsFull);
if (list != null && list.Count == maxSamples) {
keys.Add(key);
samples += maxSamples;
@ -337,7 +367,7 @@ public class OpenSeeExpression : MonoBehaviour
classLabels = keys.ToArray();
int train_split = maxSamples * 3 / 4;
int test_split = maxSamples - train_split;
int rows_train = classes * train_split;
int rows_train = (classes + 2) * train_split;
int rows_test = classes * test_split;
float[] X_train = new float[rows_train * cols];
float[] X_test = new float[rows_test * cols];
@ -347,7 +377,8 @@ public class OpenSeeExpression : MonoBehaviour
int i_test = 0;
System.Random rnd = new System.Random();
for (int i = 0; i < classes; i++) {
List<float[]> list = expressions[keys[i]];
List<float[]> list = new List<float[]>(expressions[keys[i]]);
list.RemoveAll(x => x.Length != colsFull);
int local_train_split = list.Count * 3 / 4;
test_split = list.Count - local_train_split;
for (int j = list.Count; j > 1;) {
@ -357,16 +388,28 @@ public class OpenSeeExpression : MonoBehaviour
list[k] = list[j];
list[j]= tmp;
}
for (int j = 0; j < train_split; j++) {
int train_split_current = train_split;
if (classLabels[i] == "neutral")
train_split_current *= 3;
for (int j = 0; j < train_split_current; j++) {
float factor = 1f;
float adder = 0f;
if (j > train_split || j > list.Count) {
factor = UnityEngine.Random.Range(0.98f, 1.02f);
adder = UnityEngine.Random.Range(-0.02f, 0.02f);
}
for (int k = 0; k < cols; k++) {
X_train[i_train * cols + k] = list[j % local_train_split][indices[k]];
float v = list[j % local_train_split][indices[k]] * factor + adder;
X_train[i_train * cols + k] = v;
}
y_train[i_train] = i;
i_train++;
}
for (int j = local_train_split; j < local_train_split + test_split; j++) {
for (int k = 0; k < cols; k++)
X_test[i_test * cols + k] = list[j][indices[k]];
for (int k = 0; k < cols; k++) {
float v = list[j][indices[k]];
X_test[i_test * cols + k] = v;
}
y_test[i_test] = i;
i_test++;
}
@ -374,7 +417,7 @@ public class OpenSeeExpression : MonoBehaviour
int probability = 0;
if (enableProbabilityTraining)
probability = 1;
model.TrainModel(X_train, y_train, rows_train, cols, probability);
model.TrainModel(X_train, y_train, rows_train, cols, probability, C);
confusionMatrix = model.ConfusionMatrix(X_test, y_test, i_test, out accuracy);
confusionMatrixString = SVMModel.FormatMatrix(confusionMatrix, classLabels);
for (int label = 0; label < classes; label++) {

8
Unity/Plugins/SVMModel.cs
View file

@ -118,10 +118,10 @@ public class SVMModel {
}
// features is a rows*cols long float array with rows written one after another. labels is a rows long float array with the corresponding classes as integral numbers from 0 to num_classes-1, where all classes have to exist. Training on more than 10000 rows is disabled.
public bool TrainModel(float[] features, float[] labels, int rows, int cols, int probability) {
public bool TrainModel(float[] features, float[] labels, int rows, int cols, int probability, float C) {
DestroyModel();
if (rows > 10000)
return false;
//if (rows > 10000)
// return false;
if (cols < 1 || rows < 1 || features.Length < rows * cols || labels.Length < rows)
return false;
@ -143,7 +143,7 @@ public class SVMModel {
maxClasses = max + 1;
this.cols = cols;
model = trainModel(features, labels, rows, cols, maxClasses, probability, 2f);
model = trainModel(features, labels, rows, cols, maxClasses, probability, C);
haveModel = true;
return true;