copilot/OpenSeeFace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

OpenSeeExpression now supports detecting expressions on point subsets.

Browse source
This commit is contained in:
Emiliana 2020-02-13 21:15:19 +01:00
parent de16fc9ff9
commit 9abc3029e7
1 changed files with 136 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

145
Unity/OpenSeeExpression.cs
View file

@ -40,6 +40,9 @@ public class OpenSeeExpression : MonoBehaviour
public bool load = false;
[Tooltip("When enabled, the expression data will be saved to the specified filename and this flag is set back to false.")]
public bool save = false;
[Header("Point selection")]
[Tooltip("These settings can be used to train an expression detection model on a subset of points.")]
public PointSelection pointSelection;
[Header("Information")]
[Tooltip("This is the of expressions for which calibration data was collected. The maximum number is 10.")]
public int expressionNumber = 0;
@ -61,13 +64,36 @@ public class OpenSeeExpression : MonoBehaviour
[Tooltip("This shows any accuracy warnings that resulted from the last training run.")]
public string[] warnings = null;
[Serializable]
public class PointSelection {
[Tooltip("When enabled, the points of the face contour will be used to determine the expression.")]
public bool pointsFaceContour = true;
[Tooltip("When enabled, the points of the right brow will be used to determine the expression.")]
public bool pointsBrowRight = true;
[Tooltip("When enabled, the points of the left brow will be used to determine the expression.")]
public bool pointsBrowLeft = true;
[Tooltip("When enabled, the points of the right eye will be used to determine the expression.")]
public bool pointsEyeRight = true;
[Tooltip("When enabled, the points of the left eye will be used to determine the expression.")]
public bool pointsEyeLeft = true;
[Tooltip("When enabled, the points of the nose will be used to determine the expression.")]
public bool pointsNose = true;
[Tooltip("When enabled, the points of the corners of the mouth will be used to determine the expression.")]
public bool pointsMouthCorner = true;
[Tooltip("When enabled, the points of the upper lip will be used to determine the expression.")]
public bool pointsLipUpper = true;
[Tooltip("When enabled, the points of the lower lip will be used to determine the expression.")]
public bool pointsLipLower = true;
}
[Serializable]
private class OpenSeeExpressionRepresentation {
private Dictionary<string, List<float[]>> expressions;
private byte[] modelBytes = null;
private string[] classLabels = null;
private int[] indices = null;
private PointSelection pointSelection;
static public void LoadSerialized(byte[] modelBytes, out Dictionary<string, List<float[]>> expressions, out SVMModel model, out string[] classLabels) {
static public void LoadSerialized(byte[] modelBytes, out Dictionary<string, List<float[]>> expressions, out SVMModel model, out string[] classLabels, out int[] indices, out PointSelection pointSelection) {
IFormatter formatter = new BinaryFormatter();
MemoryStream memoryStream = new MemoryStream(modelBytes);
memoryStream.Position = 0;
@ -78,13 +104,24 @@ public class OpenSeeExpression : MonoBehaviour
expressions = oser.expressions;
model = new SVMModel(oser.modelBytes);
classLabels = oser.classLabels;
indices = oser.indices;
pointSelection = oser.pointSelection;
if (indices == null) {
indices = new int[1 + 1 + 3 + 4 + 3 + 3 * 66];
for (int i = 0; i < 1 + 1 + 3 + 4 + 3 + 3 * 66; i++)
indices[i] = i;
}
if (pointSelection == null)
pointSelection = new PointSelection();
}
static public byte[] ToSerialized(Dictionary<string, List<float[]>> expressions, SVMModel model, string[] classLabels) {
static public byte[] ToSerialized(Dictionary<string, List<float[]>> expressions, SVMModel model, string[] classLabels, int[] indices, PointSelection pointSelection) {
OpenSeeExpressionRepresentation oser = new OpenSeeExpressionRepresentation();
oser.expressions = expressions;
oser.modelBytes = model.SaveModel();
oser.classLabels = classLabels;
oser.indices = indices;
oser.pointSelection = pointSelection;
IFormatter formatter = new BinaryFormatter();
MemoryStream memoryStream = new MemoryStream();
@ -101,7 +138,9 @@ public class OpenSeeExpression : MonoBehaviour
private string[] classLabels = null;
private int maxSamples = 1000; // Allows 10 expressions without exceeding 10000 rows in total for SVM training, which is a commonly seen upper limit.
// rightEyeOpen, leftEyeOpen, translation, rawQuaternion, rawEuler, confidence, points/(width, height), points3D
private int cols = 1 + 1 + 3 + 4 + 3 + /*66 + 2 * 66 +*/ 3 * 66;
private int colsFull = 1 + 1 + 3 + 4 + 3 + /*66 + 2 * 66 +*/ 3 * 66;
private int colsBase = 1 + 1 + 3 + 4 + 3;
private int cols;
private double lastCapture = 0.0;
private float warningThreshold = 5;
private int currentPrediction = -1;
@ -110,6 +149,89 @@ public class OpenSeeExpression : MonoBehaviour
private int frameCount = 0;
private System.Random rnd;
private int[] indices;
private int[] indicesFaceContour = new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
private int[] indicesBrowRight = new int[] {17, 18, 19, 20, 21};
private int[] indicesBrowLeft = new int[] {22, 23, 24, 25, 26};
private int[] indicesEyeRight = new int[] {36, 37, 38, 39, 40, 41};
private int[] indicesEyeLeft = new int[] {42, 43, 44, 45, 46, 47};
private int[] indicesNose = new int[] {27, 28, 29, 30, 31, 32, 33, 34, 35};
private int[] indicesMouthCorner = new int[] {58, 62};
private int[] indicesLipUpper = new int[] {48, 49, 50, 51, 52, 59, 60, 61};
private int[] indicesLipLower = new int[] {53, 54, 55, 56, 57, 63, 64, 65};
private void SelectPoints() {
List<int> indexList = new List<int>();
for (int i = 0; i < colsBase; i++)
indexList.Add(i);
cols = colsBase;
if (pointSelection.pointsFaceContour)
foreach (int i in indicesFaceContour) {
indexList.Add(colsBase + i * 3);
indexList.Add(colsBase + i * 3 + 1);
indexList.Add(colsBase + i * 3 + 2);
cols += 3;
}
if (pointSelection.pointsBrowRight)
foreach (int i in indicesBrowRight) {
indexList.Add(colsBase + i * 3);
indexList.Add(colsBase + i * 3 + 1);
indexList.Add(colsBase + i * 3 + 2);
cols += 3;
}
if (pointSelection.pointsBrowLeft)
foreach (int i in indicesBrowLeft) {
indexList.Add(colsBase + i * 3);
indexList.Add(colsBase + i * 3 + 1);
indexList.Add(colsBase + i * 3 + 2);
cols += 3;
}
if (pointSelection.pointsEyeRight)
foreach (int i in indicesEyeRight) {
indexList.Add(colsBase + i * 3);
indexList.Add(colsBase + i * 3 + 1);
indexList.Add(colsBase + i * 3 + 2);
cols += 3;
}
if (pointSelection.pointsEyeLeft)
foreach (int i in indicesEyeLeft) {
indexList.Add(colsBase + i * 3);
indexList.Add(colsBase + i * 3 + 1);
indexList.Add(colsBase + i * 3 + 2);
cols += 3;
}
if (pointSelection.pointsNose)
foreach (int i in indicesNose) {
indexList.Add(colsBase + i * 3);
indexList.Add(colsBase + i * 3 + 1);
indexList.Add(colsBase + i * 3 + 2);
cols += 3;
}
if (pointSelection.pointsNose)
foreach (int i in indicesMouthCorner) {
indexList.Add(colsBase + i * 3);
indexList.Add(colsBase + i * 3 + 1);
indexList.Add(colsBase + i * 3 + 2);
cols += 3;
}
if (pointSelection.pointsLipUpper)
foreach (int i in indicesLipUpper) {
indexList.Add(colsBase + i * 3);
indexList.Add(colsBase + i * 3 + 1);
indexList.Add(colsBase + i * 3 + 2);
cols += 3;
}
if (pointSelection.pointsLipLower)
foreach (int i in indicesLipLower) {
indexList.Add(colsBase + i * 3);
indexList.Add(colsBase + i * 3 + 1);
indexList.Add(colsBase + i * 3 + 2);
cols += 3;
}
indices = indexList.ToArray();
}
private void ResetInfo() {
recording = false;
clear = false;
@ -142,7 +264,7 @@ public class OpenSeeExpression : MonoBehaviour
}
private float[] GetData(OpenSee.OpenSeeData t) {
float[] data = new float[cols];
float[] data = new float[colsFull];
data[0] = t.rightEyeOpen;
data[1] = t.leftEyeOpen;
data[2] = t.translation.x;
@ -175,6 +297,7 @@ public class OpenSeeExpression : MonoBehaviour
ResetInfo();
return false;
}
SelectPoints();
List<string> keys = new List<string>();
List<string> accuracyWarnings = new List<string>();
int samples = 0;
@ -227,14 +350,14 @@ public class OpenSeeExpression : MonoBehaviour
}
for (int j = 0; j < train_split; j++) {
for (int k = 0; k < cols; k++) {
X_train[i_train * cols + k] = list[j % local_train_split][k];
X_train[i_train * cols + k] = list[j % local_train_split][indices[k]];
}
y_train[i_train] = i;
i_train++;
}
for (int j = local_train_split; j < local_train_split + test_split; j++) {
for (int k = 0; k < cols; k++)
X_test[i_test * cols + k] = list[j][k];
X_test[i_test * cols + k] = list[j][indices[k]];
y_test[i_test] = i;
i_test++;
}
@ -273,7 +396,10 @@ public class OpenSeeExpression : MonoBehaviour
if (t == null || t.time <= lastCapture)
return false;
float[] faceData = GetData(t);
float[] prediction = model.Predict(faceData, 1);
float[] predictionData = new float[cols];
for (int i = 0; i < cols; i++)
predictionData[i] = faceData[indices[i]];
float[] prediction = model.Predict(predictionData, 1);
int predictedExpression = (int)Mathf.Round(prediction[0]);
if (predictedExpression == lastPrediction)
lastPredictionCount++;
@ -359,7 +485,8 @@ public class OpenSeeExpression : MonoBehaviour
ResetInfo();
return;
}
OpenSeeExpressionRepresentation.LoadSerialized(data, out expressions, out model, out classLabels);
OpenSeeExpressionRepresentation.LoadSerialized(data, out expressions, out model, out classLabels, out indices, out pointSelection);
cols = indices.Length;
if (model.Ready())
modelReady = true;
}
@ -370,7 +497,7 @@ public class OpenSeeExpression : MonoBehaviour
ResetInfo();
return null;
}
return OpenSeeExpressionRepresentation.ToSerialized(expressions, model, classLabels);
return OpenSeeExpressionRepresentation.ToSerialized(expressions, model, classLabels, indices, pointSelection);
}
public void LoadFromFile() {