copilot/OpenSeeFace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

It is now possible to make certain expressions more important when training SVM models.

Browse source
This commit is contained in:
Emiliana 2020-03-07 13:35:58 +01:00
parent 94d2f4fbe0
commit 9aeb4fe95d
5 changed files with 56 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

28
Examples/OpenSeeVRMDriver.cs
View file

@ -26,6 +26,8 @@ public class OpenSeeVRMDriver : MonoBehaviour {
//public bool fixedUpdate = true; //public bool fixedUpdate = true;
[Tooltip("When enabled, the avatar will blink automatically. Otherwise, blinks will be applied according to the face tracking's blink detection, allowing it to wink. Even for tracked eye blinks, the eye blink setting of the current expression is used.")] [Tooltip("When enabled, the avatar will blink automatically. Otherwise, blinks will be applied according to the face tracking's blink detection, allowing it to wink. Even for tracked eye blinks, the eye blink setting of the current expression is used.")]
public bool autoBlink = true; public bool autoBlink = true;
[Tooltip("When automatic eye blinks are disabled, this sets the threshold for how closed the eyes have to be to be registered as closed. At 0 the eyes will never close.")]
public float eyeClosedThreshold = 0.3f;
[Tooltip("When enabled, the avatar's eye will move according to the face tracker's gaze tracking.")] [Tooltip("When enabled, the avatar's eye will move according to the face tracker's gaze tracking.")]
public bool gazeTracking = true; public bool gazeTracking = true;
[Tooltip("This is the right eye bone. When either eye bone is not set, the VRM look direction blendshapes are used instead.")] [Tooltip("This is the right eye bone. When either eye bone is not set, the VRM look direction blendshapes are used instead.")]
@ -44,12 +46,12 @@ public class OpenSeeVRMDriver : MonoBehaviour {
public OpenSeeEyeBlink eyeBlinker = new OpenSeeEyeBlink(); public OpenSeeEyeBlink eyeBlinker = new OpenSeeEyeBlink();
[Tooltip("This component lets configure your VRM expressions.")] [Tooltip("This component lets configure your VRM expressions.")]
public OpenSeeVRMExpression[] expressions = new OpenSeeVRMExpression[]{ public OpenSeeVRMExpression[] expressions = new OpenSeeVRMExpression[]{
new OpenSeeVRMExpression("neutral", BlendShapePreset.Neutral, 1f, 1f, true, true, 0x70), new OpenSeeVRMExpression("neutral", BlendShapePreset.Neutral, 1f, 1f, true, true, 0x70, 50f),
new OpenSeeVRMExpression("fun", BlendShapePreset.Fun, 1f, 1f, true, true, 0x71), new OpenSeeVRMExpression("fun", BlendShapePreset.Fun, 1f, 1f, true, true, 0x71, 1f),
new OpenSeeVRMExpression("joy", BlendShapePreset.Joy, 1f, 1f, true, true, 0x72), new OpenSeeVRMExpression("joy", BlendShapePreset.Joy, 1f, 1f, true, true, 0x72, 1f),
new OpenSeeVRMExpression("angry", BlendShapePreset.Angry, 1f, 1f, true, true, 0x73), new OpenSeeVRMExpression("angry", BlendShapePreset.Angry, 1f, 1f, true, true, 0x73, 1f),
new OpenSeeVRMExpression("sorrow", BlendShapePreset.Sorrow, 1f, 1f, true, true, 0x74), new OpenSeeVRMExpression("sorrow", BlendShapePreset.Sorrow, 1f, 1f, true, true, 0x74, 1f),
new OpenSeeVRMExpression("surprise", "Surprised", 1f, 1f, true, true, 0x75) new OpenSeeVRMExpression("surprise", "Surprised", 1f, 1f, true, true, 0x75, 1f)
}; };
[Tooltip("The expression configuration is initialized on startup. If it is changed and needs to be reinitialized, this can be triggered by using this flag or calling InitExpressionMap. This flag is reset to false afterwards.")] [Tooltip("The expression configuration is initialized on startup. If it is changed and needs to be reinitialized, this can be triggered by using this flag or calling InitExpressionMap. This flag is reset to false afterwards.")]
public bool reloadExpressions = false; public bool reloadExpressions = false;
@ -184,10 +186,12 @@ public class OpenSeeVRMDriver : MonoBehaviour {
public bool enableBlinking = true; public bool enableBlinking = true;
[Tooltip("This can be set to a virtual key value. If this key is pressed together with shift and control, the expression will trigger as an override.")] [Tooltip("This can be set to a virtual key value. If this key is pressed together with shift and control, the expression will trigger as an override.")]
public int hotkey = -1; public int hotkey = -1;
[Tooltip("This is the weight for this class's error when training the model.")]
public float errorWeight = 1f;
[HideInInspector] [HideInInspector]
public BlendShapeKey blendShapeKey; public BlendShapeKey blendShapeKey;
public OpenSeeVRMExpression(string trigger, BlendShapePreset preset, float weight, float factor, bool visemes, bool blinking, int hotkey) { public OpenSeeVRMExpression(string trigger, BlendShapePreset preset, float weight, float factor, bool visemes, bool blinking, int hotkey, float errorWeight) {
this.trigger = trigger; this.trigger = trigger;
this.blendShapePreset = preset; this.blendShapePreset = preset;
this.weight = weight; this.weight = weight;
@ -195,9 +199,10 @@ public class OpenSeeVRMDriver : MonoBehaviour {
this.enableVisemes = visemes; this.enableVisemes = visemes;
this.enableBlinking = blinking; this.enableBlinking = blinking;
this.hotkey = hotkey; this.hotkey = hotkey;
this.errorWeight = errorWeight;
} }
public OpenSeeVRMExpression(string trigger, string name, float weight, float factor, bool visemes, bool blinking, int hotkey) { public OpenSeeVRMExpression(string trigger, string name, float weight, float factor, bool visemes, bool blinking, int hotkey, float errorWeight) {
this.trigger = trigger; this.trigger = trigger;
this.customBlendShapeName = name; this.customBlendShapeName = name;
this.weight = weight; this.weight = weight;
@ -205,12 +210,15 @@ public class OpenSeeVRMDriver : MonoBehaviour {
this.enableVisemes = visemes; this.enableVisemes = visemes;
this.enableBlinking = blinking; this.enableBlinking = blinking;
this.hotkey = hotkey; this.hotkey = hotkey;
this.errorWeight = errorWeight;
} }
} }
public void InitExpressionMap() { public void InitExpressionMap() {
expressionMap = new Dictionary<string, OpenSeeVRMExpression>(); expressionMap = new Dictionary<string, OpenSeeVRMExpression>();
openSeeExpression.weightMap = new Dictionary<string, float>();
foreach (var expression in expressions) { foreach (var expression in expressions) {
openSeeExpression.weightMap.Add(expression.trigger, expression.errorWeight);
if (expression.customBlendShapeName != "") if (expression.customBlendShapeName != "")
expression.blendShapeKey = new BlendShapeKey(expression.customBlendShapeName); expression.blendShapeKey = new BlendShapeKey(expression.customBlendShapeName);
else else
@ -382,10 +390,10 @@ public class OpenSeeVRMDriver : MonoBehaviour {
return; return;
} }
float right = 1f; float right = 1f;
if (openSeeData.rightEyeOpen > 0.5f) if (openSeeData.rightEyeOpen > eyeClosedThreshold)
right = 0f; right = 0f;
float left = 1f; float left = 1f;
if (openSeeData.leftEyeOpen > 0.5f) if (openSeeData.leftEyeOpen > eyeClosedThreshold)
left = 0f; left = 0f;
vrmBlendShapeProxy.ImmediatelySetValue(new BlendShapeKey(BlendShapePreset.Blink_R), right); vrmBlendShapeProxy.ImmediatelySetValue(new BlendShapeKey(BlendShapePreset.Blink_R), right);
vrmBlendShapeProxy.ImmediatelySetValue(new BlendShapeKey(BlendShapePreset.Blink_L), left); vrmBlendShapeProxy.ImmediatelySetValue(new BlendShapeKey(BlendShapePreset.Blink_L), left);

17
Source/SVMWrapper.cpp
View file

@ -87,7 +87,7 @@ void destroy_problem(struct svm_problem *problem) {
} }
extern "C" { extern "C" {
__declspec(dllexport) void *__cdecl trainModel(float features[], float labels[], int rows, int cols, int classes, int probability, float C) { __declspec(dllexport) void *__cdecl trainModel(float features[], float labels[], float weights[], int rows, int cols, int classes, int probability, float C) {
if (!(rows > 0 && cols > 0)) if (!(rows > 0 && cols > 0))
return NULL; return NULL;
@ -115,10 +115,25 @@ extern "C" {
sw->param.weight_label = NULL; sw->param.weight_label = NULL;
sw->param.weight = NULL; sw->param.weight = NULL;
if (weights != NULL) {
sw->param.nr_weight = classes;
sw->param.weight_label = new int[classes];
sw->param.weight = new double[classes];
for (int i = 0; i < classes; i++) {
sw->param.weight_label[i] = i;
sw->param.weight[i] = (double)weights[i];
}
}
sw->problem = new struct svm_problem(); sw->problem = new struct svm_problem();
make_problem(sw->problem, features, labels, rows, cols, sw->means, sw->sdevs); make_problem(sw->problem, features, labels, rows, cols, sw->means, sw->sdevs);
sw->model = svm_train(sw->problem, &sw->param); sw->model = svm_train(sw->problem, &sw->param);
if (weights != NULL) {
delete sw->param.weight_label;
delete sw->param.weight;
}
return sw; return sw;
} }

25
Unity/OpenSeeExpression.cs
View file

@ -29,6 +29,8 @@ public class OpenSeeExpression : MonoBehaviour
public bool enableProbabilityTraining = false; public bool enableProbabilityTraining = false;
[Tooltip("This is the SVM training C value.")] [Tooltip("This is the SVM training C value.")]
public float C = 2f; public float C = 2f;
[Tooltip("This map allows you to set weights for different classes.")]
public Dictionary<string, float> weightMap = null;
[Header("Toggles")] [Header("Toggles")]
[Tooltip("When enabled, calibration data will be collected from the given OpenSee component.")] [Tooltip("When enabled, calibration data will be collected from the given OpenSee component.")]
public bool recording = false; public bool recording = false;
@ -341,12 +343,13 @@ public class OpenSeeExpression : MonoBehaviour
List<float[]> list = new List<float[]>(expressions[key]); List<float[]> list = new List<float[]>(expressions[key]);
list.RemoveAll(x => x.Length != colsFull); list.RemoveAll(x => x.Length != colsFull);
if (list != null && list.Count == maxSamples) { if (list != null && list.Count == maxSamples) {
Debug.Log("[Training info] Adding expression " + key + " to training data.");
keys.Add(key); keys.Add(key);
samples += maxSamples; samples += list.Count;
} else { } else {
if (list != null && list.Count > 20) { if (list != null && list.Count > 20) {
//Debug.Log("[Training warning] Expression " + key + " has little data and might be inaccurate."); Debug.Log("[Training warning] Expression " + key + " has little data and might be inaccurate.");
//accuracyWarnings.Add("Expression " + key + " has little data and might be inaccurate."); accuracyWarnings.Add("Expression " + key + " has little data and might be inaccurate.");
samples += list.Count; samples += list.Count;
keys.Add(key); keys.Add(key);
} else { } else {
@ -364,9 +367,19 @@ public class OpenSeeExpression : MonoBehaviour
} }
keys.Sort(); keys.Sort();
classLabels = keys.ToArray(); classLabels = keys.ToArray();
float[] weights = null;
if (weightMap != null) {
weights = new float[classLabels.Length];
for (int i = 0; i < classLabels.Length; i++) {
weights[i] = weightMap[classLabels[i]];
Debug.Log("[Training info] Adding weight " + weights[i] + " for " + classLabels[i] + ".");
}
}
int train_split = maxSamples * 3 / 4; int train_split = maxSamples * 3 / 4;
int test_split = maxSamples - train_split; int test_split = maxSamples - train_split;
int rows_train = (classes + 2) * train_split; int rows_train = classes * train_split;
int rows_test = classes * test_split; int rows_test = classes * test_split;
float[] X_train = new float[rows_train * cols]; float[] X_train = new float[rows_train * cols];
float[] X_test = new float[rows_test * cols]; float[] X_test = new float[rows_test * cols];
@ -389,7 +402,7 @@ public class OpenSeeExpression : MonoBehaviour
} }
int train_split_current = train_split; int train_split_current = train_split;
if (classLabels[i] == "neutral") if (classLabels[i] == "neutral")
train_split_current *= 3; train_split_current *= 1;
for (int j = 0; j < train_split_current; j++) { for (int j = 0; j < train_split_current; j++) {
float factor = 1f; float factor = 1f;
float adder = 0f; float adder = 0f;
@ -416,7 +429,7 @@ public class OpenSeeExpression : MonoBehaviour
int probability = 0; int probability = 0;
if (enableProbabilityTraining) if (enableProbabilityTraining)
probability = 1; probability = 1;
model.TrainModel(X_train, y_train, i_train, cols, probability, C); model.TrainModel(X_train, y_train, weights, i_train, cols, probability, C);
confusionMatrix = model.ConfusionMatrix(X_test, y_test, i_test, out accuracy); confusionMatrix = model.ConfusionMatrix(X_test, y_test, i_test, out accuracy);
confusionMatrixString = SVMModel.FormatMatrix(confusionMatrix, classLabels); confusionMatrixString = SVMModel.FormatMatrix(confusionMatrix, classLabels);
for (int label = 0; label < classes; label++) { for (int label = 0; label < classes; label++) {

6
Unity/Plugins/SVMModel.cs
View file

@ -12,7 +12,7 @@ namespace OpenSee {
public class SVMModel { public class SVMModel {
#region DllImport #region DllImport
[DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl)] [DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl)]
private static extern System.IntPtr trainModel(float[] features, float[] labels, int rows, int cols, int classes, int probability, float C); private static extern System.IntPtr trainModel(float[] features, float[] labels, float[] weights, int rows, int cols, int classes, int probability, float C);
[DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl)] [DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl)]
private static extern void predict(System.IntPtr model, float[] features, [Out] float[] predictions, [Out] double[] probabilities, int rows); private static extern void predict(System.IntPtr model, float[] features, [Out] float[] predictions, [Out] double[] probabilities, int rows);
@ -118,7 +118,7 @@ public class SVMModel {
} }
// features is a rows*cols long float array with rows written one after another. labels is a rows long float array with the corresponding classes as integral numbers from 0 to num_classes-1, where all classes have to exist. Training on more than 10000 rows is disabled. // features is a rows*cols long float array with rows written one after another. labels is a rows long float array with the corresponding classes as integral numbers from 0 to num_classes-1, where all classes have to exist. Training on more than 10000 rows is disabled.
public bool TrainModel(float[] features, float[] labels, int rows, int cols, int probability, float C) { public bool TrainModel(float[] features, float[] labels, float[] weights, int rows, int cols, int probability, float C) {
DestroyModel(); DestroyModel();
//if (rows > 10000) //if (rows > 10000)
// return false; // return false;
@ -143,7 +143,7 @@ public class SVMModel {
maxClasses = max + 1; maxClasses = max + 1;
this.cols = cols; this.cols = cols;
model = trainModel(features, labels, rows, cols, maxClasses, probability, C); model = trainModel(features, labels, weights, rows, cols, maxClasses, probability, C);
haveModel = true; haveModel = true;
return true; return true;

BIN
Unity/Plugins/SVMModel.dll
View file

Binary file not shown.