From 9abc3029e71f17b29326e6cc180820b251db7b4c Mon Sep 17 00:00:00 2001 From: Emiliana <38952746+emilianavt@users.noreply.github.com> Date: Thu, 13 Feb 2020 21:15:19 +0100 Subject: [PATCH] OpenSeeExpression now supports detecting expressions on point subsets. --- Unity/OpenSeeExpression.cs | 145 ++++++++++++++++++++++++++++++++++--- 1 file changed, 136 insertions(+), 9 deletions(-) diff --git a/Unity/OpenSeeExpression.cs b/Unity/OpenSeeExpression.cs index 83ca321..4143d62 100644 --- a/Unity/OpenSeeExpression.cs +++ b/Unity/OpenSeeExpression.cs @@ -40,6 +40,9 @@ public class OpenSeeExpression : MonoBehaviour public bool load = false; [Tooltip("When enabled, the expression data will be saved to the specified filename and this flag is set back to false.")] public bool save = false; + [Header("Point selection")] + [Tooltip("These settings can be used to train an expression detection model on a subset of points.")] + public PointSelection pointSelection; [Header("Information")] [Tooltip("This is the of expressions for which calibration data was collected. The maximum number is 10.")] public int expressionNumber = 0; @@ -61,13 +64,36 @@ public class OpenSeeExpression : MonoBehaviour [Tooltip("This shows any accuracy warnings that resulted from the last training run.")] public string[] warnings = null; + [Serializable] + public class PointSelection { + [Tooltip("When enabled, the points of the face contour will be used to determine the expression.")] + public bool pointsFaceContour = true; + [Tooltip("When enabled, the points of the right brow will be used to determine the expression.")] + public bool pointsBrowRight = true; + [Tooltip("When enabled, the points of the left brow will be used to determine the expression.")] + public bool pointsBrowLeft = true; + [Tooltip("When enabled, the points of the right eye will be used to determine the expression.")] + public bool pointsEyeRight = true; + [Tooltip("When enabled, the points of the left eye will be used to determine the expression.")] + public bool pointsEyeLeft = true; + [Tooltip("When enabled, the points of the nose will be used to determine the expression.")] + public bool pointsNose = true; + [Tooltip("When enabled, the points of the corners of the mouth will be used to determine the expression.")] + public bool pointsMouthCorner = true; + [Tooltip("When enabled, the points of the upper lip will be used to determine the expression.")] + public bool pointsLipUpper = true; + [Tooltip("When enabled, the points of the lower lip will be used to determine the expression.")] + public bool pointsLipLower = true; + } [Serializable] private class OpenSeeExpressionRepresentation { private Dictionary> expressions; private byte[] modelBytes = null; private string[] classLabels = null; + private int[] indices = null; + private PointSelection pointSelection; - static public void LoadSerialized(byte[] modelBytes, out Dictionary> expressions, out SVMModel model, out string[] classLabels) { + static public void LoadSerialized(byte[] modelBytes, out Dictionary> expressions, out SVMModel model, out string[] classLabels, out int[] indices, out PointSelection pointSelection) { IFormatter formatter = new BinaryFormatter(); MemoryStream memoryStream = new MemoryStream(modelBytes); memoryStream.Position = 0; @@ -78,13 +104,24 @@ public class OpenSeeExpression : MonoBehaviour expressions = oser.expressions; model = new SVMModel(oser.modelBytes); classLabels = oser.classLabels; + indices = oser.indices; + pointSelection = oser.pointSelection; + if (indices == null) { + indices = new int[1 + 1 + 3 + 4 + 3 + 3 * 66]; + for (int i = 0; i < 1 + 1 + 3 + 4 + 3 + 3 * 66; i++) + indices[i] = i; + } + if (pointSelection == null) + pointSelection = new PointSelection(); } - static public byte[] ToSerialized(Dictionary> expressions, SVMModel model, string[] classLabels) { + static public byte[] ToSerialized(Dictionary> expressions, SVMModel model, string[] classLabels, int[] indices, PointSelection pointSelection) { OpenSeeExpressionRepresentation oser = new OpenSeeExpressionRepresentation(); oser.expressions = expressions; oser.modelBytes = model.SaveModel(); oser.classLabels = classLabels; + oser.indices = indices; + oser.pointSelection = pointSelection; IFormatter formatter = new BinaryFormatter(); MemoryStream memoryStream = new MemoryStream(); @@ -101,7 +138,9 @@ public class OpenSeeExpression : MonoBehaviour private string[] classLabels = null; private int maxSamples = 1000; // Allows 10 expressions without exceeding 10000 rows in total for SVM training, which is a commonly seen upper limit. // rightEyeOpen, leftEyeOpen, translation, rawQuaternion, rawEuler, confidence, points/(width, height), points3D - private int cols = 1 + 1 + 3 + 4 + 3 + /*66 + 2 * 66 +*/ 3 * 66; + private int colsFull = 1 + 1 + 3 + 4 + 3 + /*66 + 2 * 66 +*/ 3 * 66; + private int colsBase = 1 + 1 + 3 + 4 + 3; + private int cols; private double lastCapture = 0.0; private float warningThreshold = 5; private int currentPrediction = -1; @@ -110,6 +149,89 @@ public class OpenSeeExpression : MonoBehaviour private int frameCount = 0; private System.Random rnd; + private int[] indices; + + private int[] indicesFaceContour = new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; + private int[] indicesBrowRight = new int[] {17, 18, 19, 20, 21}; + private int[] indicesBrowLeft = new int[] {22, 23, 24, 25, 26}; + private int[] indicesEyeRight = new int[] {36, 37, 38, 39, 40, 41}; + private int[] indicesEyeLeft = new int[] {42, 43, 44, 45, 46, 47}; + private int[] indicesNose = new int[] {27, 28, 29, 30, 31, 32, 33, 34, 35}; + private int[] indicesMouthCorner = new int[] {58, 62}; + private int[] indicesLipUpper = new int[] {48, 49, 50, 51, 52, 59, 60, 61}; + private int[] indicesLipLower = new int[] {53, 54, 55, 56, 57, 63, 64, 65}; + + private void SelectPoints() { + List indexList = new List(); + for (int i = 0; i < colsBase; i++) + indexList.Add(i); + cols = colsBase; + if (pointSelection.pointsFaceContour) + foreach (int i in indicesFaceContour) { + indexList.Add(colsBase + i * 3); + indexList.Add(colsBase + i * 3 + 1); + indexList.Add(colsBase + i * 3 + 2); + cols += 3; + } + if (pointSelection.pointsBrowRight) + foreach (int i in indicesBrowRight) { + indexList.Add(colsBase + i * 3); + indexList.Add(colsBase + i * 3 + 1); + indexList.Add(colsBase + i * 3 + 2); + cols += 3; + } + if (pointSelection.pointsBrowLeft) + foreach (int i in indicesBrowLeft) { + indexList.Add(colsBase + i * 3); + indexList.Add(colsBase + i * 3 + 1); + indexList.Add(colsBase + i * 3 + 2); + cols += 3; + } + if (pointSelection.pointsEyeRight) + foreach (int i in indicesEyeRight) { + indexList.Add(colsBase + i * 3); + indexList.Add(colsBase + i * 3 + 1); + indexList.Add(colsBase + i * 3 + 2); + cols += 3; + } + if (pointSelection.pointsEyeLeft) + foreach (int i in indicesEyeLeft) { + indexList.Add(colsBase + i * 3); + indexList.Add(colsBase + i * 3 + 1); + indexList.Add(colsBase + i * 3 + 2); + cols += 3; + } + if (pointSelection.pointsNose) + foreach (int i in indicesNose) { + indexList.Add(colsBase + i * 3); + indexList.Add(colsBase + i * 3 + 1); + indexList.Add(colsBase + i * 3 + 2); + cols += 3; + } + if (pointSelection.pointsNose) + foreach (int i in indicesMouthCorner) { + indexList.Add(colsBase + i * 3); + indexList.Add(colsBase + i * 3 + 1); + indexList.Add(colsBase + i * 3 + 2); + cols += 3; + } + if (pointSelection.pointsLipUpper) + foreach (int i in indicesLipUpper) { + indexList.Add(colsBase + i * 3); + indexList.Add(colsBase + i * 3 + 1); + indexList.Add(colsBase + i * 3 + 2); + cols += 3; + } + if (pointSelection.pointsLipLower) + foreach (int i in indicesLipLower) { + indexList.Add(colsBase + i * 3); + indexList.Add(colsBase + i * 3 + 1); + indexList.Add(colsBase + i * 3 + 2); + cols += 3; + } + indices = indexList.ToArray(); + } + private void ResetInfo() { recording = false; clear = false; @@ -142,7 +264,7 @@ public class OpenSeeExpression : MonoBehaviour } private float[] GetData(OpenSee.OpenSeeData t) { - float[] data = new float[cols]; + float[] data = new float[colsFull]; data[0] = t.rightEyeOpen; data[1] = t.leftEyeOpen; data[2] = t.translation.x; @@ -175,6 +297,7 @@ public class OpenSeeExpression : MonoBehaviour ResetInfo(); return false; } + SelectPoints(); List keys = new List(); List accuracyWarnings = new List(); int samples = 0; @@ -227,14 +350,14 @@ public class OpenSeeExpression : MonoBehaviour } for (int j = 0; j < train_split; j++) { for (int k = 0; k < cols; k++) { - X_train[i_train * cols + k] = list[j % local_train_split][k]; + X_train[i_train * cols + k] = list[j % local_train_split][indices[k]]; } y_train[i_train] = i; i_train++; } for (int j = local_train_split; j < local_train_split + test_split; j++) { for (int k = 0; k < cols; k++) - X_test[i_test * cols + k] = list[j][k]; + X_test[i_test * cols + k] = list[j][indices[k]]; y_test[i_test] = i; i_test++; } @@ -273,7 +396,10 @@ public class OpenSeeExpression : MonoBehaviour if (t == null || t.time <= lastCapture) return false; float[] faceData = GetData(t); - float[] prediction = model.Predict(faceData, 1); + float[] predictionData = new float[cols]; + for (int i = 0; i < cols; i++) + predictionData[i] = faceData[indices[i]]; + float[] prediction = model.Predict(predictionData, 1); int predictedExpression = (int)Mathf.Round(prediction[0]); if (predictedExpression == lastPrediction) lastPredictionCount++; @@ -359,7 +485,8 @@ public class OpenSeeExpression : MonoBehaviour ResetInfo(); return; } - OpenSeeExpressionRepresentation.LoadSerialized(data, out expressions, out model, out classLabels); + OpenSeeExpressionRepresentation.LoadSerialized(data, out expressions, out model, out classLabels, out indices, out pointSelection); + cols = indices.Length; if (model.Ready()) modelReady = true; } @@ -370,7 +497,7 @@ public class OpenSeeExpression : MonoBehaviour ResetInfo(); return null; } - return OpenSeeExpressionRepresentation.ToSerialized(expressions, model, classLabels); + return OpenSeeExpressionRepresentation.ToSerialized(expressions, model, classLabels, indices, pointSelection); } public void LoadFromFile() {