copilot/OpenSeeFace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

It is now possible to make certain expressions more important when training SVM models.

Browse source
This commit is contained in:
Emiliana 2020-03-07 13:35:58 +01:00
parent 94d2f4fbe0
commit 9aeb4fe95d
5 changed files with 56 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

28
Examples/OpenSeeVRMDriver.cs
View file

@ -26,6 +26,8 @@ public class OpenSeeVRMDriver : MonoBehaviour {
//public bool fixedUpdate = true;
[Tooltip("When enabled, the avatar will blink automatically. Otherwise, blinks will be applied according to the face tracking's blink detection, allowing it to wink. Even for tracked eye blinks, the eye blink setting of the current expression is used.")]
public bool autoBlink = true;
[Tooltip("When automatic eye blinks are disabled, this sets the threshold for how closed the eyes have to be to be registered as closed. At 0 the eyes will never close.")]
public float eyeClosedThreshold = 0.3f;
[Tooltip("When enabled, the avatar's eye will move according to the face tracker's gaze tracking.")]
public bool gazeTracking = true;
[Tooltip("This is the right eye bone. When either eye bone is not set, the VRM look direction blendshapes are used instead.")]
@ -44,12 +46,12 @@ public class OpenSeeVRMDriver : MonoBehaviour {
public OpenSeeEyeBlink eyeBlinker = new OpenSeeEyeBlink();
[Tooltip("This component lets configure your VRM expressions.")]
public OpenSeeVRMExpression[] expressions = new OpenSeeVRMExpression[]{
new OpenSeeVRMExpression("neutral", BlendShapePreset.Neutral, 1f, 1f, true, true, 0x70),
new OpenSeeVRMExpression("fun", BlendShapePreset.Fun, 1f, 1f, true, true, 0x71),
new OpenSeeVRMExpression("joy", BlendShapePreset.Joy, 1f, 1f, true, true, 0x72),
new OpenSeeVRMExpression("angry", BlendShapePreset.Angry, 1f, 1f, true, true, 0x73),
new OpenSeeVRMExpression("sorrow", BlendShapePreset.Sorrow, 1f, 1f, true, true, 0x74),
new OpenSeeVRMExpression("surprise", "Surprised", 1f, 1f, true, true, 0x75)
new OpenSeeVRMExpression("neutral", BlendShapePreset.Neutral, 1f, 1f, true, true, 0x70, 50f),
new OpenSeeVRMExpression("fun", BlendShapePreset.Fun, 1f, 1f, true, true, 0x71, 1f),
new OpenSeeVRMExpression("joy", BlendShapePreset.Joy, 1f, 1f, true, true, 0x72, 1f),
new OpenSeeVRMExpression("angry", BlendShapePreset.Angry, 1f, 1f, true, true, 0x73, 1f),
new OpenSeeVRMExpression("sorrow", BlendShapePreset.Sorrow, 1f, 1f, true, true, 0x74, 1f),
new OpenSeeVRMExpression("surprise", "Surprised", 1f, 1f, true, true, 0x75, 1f)
};
[Tooltip("The expression configuration is initialized on startup. If it is changed and needs to be reinitialized, this can be triggered by using this flag or calling InitExpressionMap. This flag is reset to false afterwards.")]
public bool reloadExpressions = false;
@ -184,10 +186,12 @@ public class OpenSeeVRMDriver : MonoBehaviour {
public bool enableBlinking = true;
[Tooltip("This can be set to a virtual key value. If this key is pressed together with shift and control, the expression will trigger as an override.")]
public int hotkey = -1;
[Tooltip("This is the weight for this class's error when training the model.")]
public float errorWeight = 1f;
[HideInInspector]
public BlendShapeKey blendShapeKey;
public OpenSeeVRMExpression(string trigger, BlendShapePreset preset, float weight, float factor, bool visemes, bool blinking, int hotkey) {
public OpenSeeVRMExpression(string trigger, BlendShapePreset preset, float weight, float factor, bool visemes, bool blinking, int hotkey, float errorWeight) {
this.trigger = trigger;
this.blendShapePreset = preset;
this.weight = weight;
@ -195,9 +199,10 @@ public class OpenSeeVRMDriver : MonoBehaviour {
this.enableVisemes = visemes;
this.enableBlinking = blinking;
this.hotkey = hotkey;
this.errorWeight = errorWeight;
}
public OpenSeeVRMExpression(string trigger, string name, float weight, float factor, bool visemes, bool blinking, int hotkey) {
public OpenSeeVRMExpression(string trigger, string name, float weight, float factor, bool visemes, bool blinking, int hotkey, float errorWeight) {
this.trigger = trigger;
this.customBlendShapeName = name;
this.weight = weight;
@ -205,12 +210,15 @@ public class OpenSeeVRMDriver : MonoBehaviour {
this.enableVisemes = visemes;
this.enableBlinking = blinking;
this.hotkey = hotkey;
this.errorWeight = errorWeight;
}
}
public void InitExpressionMap() {
expressionMap = new Dictionary<string, OpenSeeVRMExpression>();
openSeeExpression.weightMap = new Dictionary<string, float>();
foreach (var expression in expressions) {
openSeeExpression.weightMap.Add(expression.trigger, expression.errorWeight);
if (expression.customBlendShapeName != "")
expression.blendShapeKey = new BlendShapeKey(expression.customBlendShapeName);
else
@ -382,10 +390,10 @@ public class OpenSeeVRMDriver : MonoBehaviour {
return;
}
float right = 1f;
if (openSeeData.rightEyeOpen > 0.5f)
if (openSeeData.rightEyeOpen > eyeClosedThreshold)
right = 0f;
float left = 1f;
if (openSeeData.leftEyeOpen > 0.5f)
if (openSeeData.leftEyeOpen > eyeClosedThreshold)
left = 0f;
vrmBlendShapeProxy.ImmediatelySetValue(new BlendShapeKey(BlendShapePreset.Blink_R), right);
vrmBlendShapeProxy.ImmediatelySetValue(new BlendShapeKey(BlendShapePreset.Blink_L), left);

17
Source/SVMWrapper.cpp
View file

@ -87,7 +87,7 @@ void destroy_problem(struct svm_problem *problem) {
}
extern "C" {
__declspec(dllexport) void *__cdecl trainModel(float features[], float labels[], int rows, int cols, int classes, int probability, float C) {
__declspec(dllexport) void *__cdecl trainModel(float features[], float labels[], float weights[], int rows, int cols, int classes, int probability, float C) {
if (!(rows > 0 && cols > 0))
return NULL;
@ -115,10 +115,25 @@ extern "C" {
sw->param.weight_label = NULL;
sw->param.weight = NULL;
if (weights != NULL) {
sw->param.nr_weight = classes;
sw->param.weight_label = new int[classes];
sw->param.weight = new double[classes];
for (int i = 0; i < classes; i++) {
sw->param.weight_label[i] = i;
sw->param.weight[i] = (double)weights[i];
}
}
sw->problem = new struct svm_problem();
make_problem(sw->problem, features, labels, rows, cols, sw->means, sw->sdevs);
sw->model = svm_train(sw->problem, &sw->param);
if (weights != NULL) {
delete sw->param.weight_label;
delete sw->param.weight;
}
return sw;
}

25
Unity/OpenSeeExpression.cs
View file

@ -29,6 +29,8 @@ public class OpenSeeExpression : MonoBehaviour
public bool enableProbabilityTraining = false;
[Tooltip("This is the SVM training C value.")]
public float C = 2f;
[Tooltip("This map allows you to set weights for different classes.")]
public Dictionary<string, float> weightMap = null;
[Header("Toggles")]
[Tooltip("When enabled, calibration data will be collected from the given OpenSee component.")]
public bool recording = false;
@ -341,12 +343,13 @@ public class OpenSeeExpression : MonoBehaviour
List<float[]> list = new List<float[]>(expressions[key]);
list.RemoveAll(x => x.Length != colsFull);
if (list != null && list.Count == maxSamples) {
Debug.Log("[Training info] Adding expression " + key + " to training data.");
keys.Add(key);
samples += maxSamples;
samples += list.Count;
} else {
if (list != null && list.Count > 20) {
//Debug.Log("[Training warning] Expression " + key + " has little data and might be inaccurate.");
//accuracyWarnings.Add("Expression " + key + " has little data and might be inaccurate.");
Debug.Log("[Training warning] Expression " + key + " has little data and might be inaccurate.");
accuracyWarnings.Add("Expression " + key + " has little data and might be inaccurate.");
samples += list.Count;
keys.Add(key);
} else {
@ -364,9 +367,19 @@ public class OpenSeeExpression : MonoBehaviour
}
keys.Sort();
classLabels = keys.ToArray();
float[] weights = null;
if (weightMap != null) {
weights = new float[classLabels.Length];
for (int i = 0; i < classLabels.Length; i++) {
weights[i] = weightMap[classLabels[i]];
Debug.Log("[Training info] Adding weight " + weights[i] + " for " + classLabels[i] + ".");
}
}
int train_split = maxSamples * 3 / 4;
int test_split = maxSamples - train_split;
int rows_train = (classes + 2) * train_split;
int rows_train = classes * train_split;
int rows_test = classes * test_split;
float[] X_train = new float[rows_train * cols];
float[] X_test = new float[rows_test * cols];
@ -389,7 +402,7 @@ public class OpenSeeExpression : MonoBehaviour
}
int train_split_current = train_split;
if (classLabels[i] == "neutral")
train_split_current *= 3;
train_split_current *= 1;
for (int j = 0; j < train_split_current; j++) {
float factor = 1f;
float adder = 0f;
@ -416,7 +429,7 @@ public class OpenSeeExpression : MonoBehaviour
int probability = 0;
if (enableProbabilityTraining)
probability = 1;
model.TrainModel(X_train, y_train, i_train, cols, probability, C);
model.TrainModel(X_train, y_train, weights, i_train, cols, probability, C);
confusionMatrix = model.ConfusionMatrix(X_test, y_test, i_test, out accuracy);
confusionMatrixString = SVMModel.FormatMatrix(confusionMatrix, classLabels);
for (int label = 0; label < classes; label++) {

6
Unity/Plugins/SVMModel.cs
View file

@ -12,7 +12,7 @@ namespace OpenSee {
public class SVMModel {
#region DllImport
[DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl)]
private static extern System.IntPtr trainModel(float[] features, float[] labels, int rows, int cols, int classes, int probability, float C);
private static extern System.IntPtr trainModel(float[] features, float[] labels, float[] weights, int rows, int cols, int classes, int probability, float C);
[DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl)]
private static extern void predict(System.IntPtr model, float[] features, [Out] float[] predictions, [Out] double[] probabilities, int rows);
@ -118,7 +118,7 @@ public class SVMModel {
}
// features is a rows*cols long float array with rows written one after another. labels is a rows long float array with the corresponding classes as integral numbers from 0 to num_classes-1, where all classes have to exist. Training on more than 10000 rows is disabled.
public bool TrainModel(float[] features, float[] labels, int rows, int cols, int probability, float C) {
public bool TrainModel(float[] features, float[] labels, float[] weights, int rows, int cols, int probability, float C) {
DestroyModel();
//if (rows > 10000)
// return false;
@ -143,7 +143,7 @@ public class SVMModel {
maxClasses = max + 1;
this.cols = cols;
model = trainModel(features, labels, rows, cols, maxClasses, probability, C);
model = trainModel(features, labels, weights, rows, cols, maxClasses, probability, C);
haveModel = true;
return true;

BIN
Unity/Plugins/SVMModel.dll
View file

Binary file not shown.