diff --git a/Examples/OpenSeeVRMDriver.cs b/Examples/OpenSeeVRMDriver.cs index d0fcf78..6a8d369 100644 --- a/Examples/OpenSeeVRMDriver.cs +++ b/Examples/OpenSeeVRMDriver.cs @@ -26,6 +26,8 @@ public class OpenSeeVRMDriver : MonoBehaviour { //public bool fixedUpdate = true; [Tooltip("When enabled, the avatar will blink automatically. Otherwise, blinks will be applied according to the face tracking's blink detection, allowing it to wink. Even for tracked eye blinks, the eye blink setting of the current expression is used.")] public bool autoBlink = true; + [Tooltip("When automatic eye blinks are disabled, this sets the threshold for how closed the eyes have to be to be registered as closed. At 0 the eyes will never close.")] + public float eyeClosedThreshold = 0.3f; [Tooltip("When enabled, the avatar's eye will move according to the face tracker's gaze tracking.")] public bool gazeTracking = true; [Tooltip("This is the right eye bone. When either eye bone is not set, the VRM look direction blendshapes are used instead.")] @@ -44,12 +46,12 @@ public class OpenSeeVRMDriver : MonoBehaviour { public OpenSeeEyeBlink eyeBlinker = new OpenSeeEyeBlink(); [Tooltip("This component lets configure your VRM expressions.")] public OpenSeeVRMExpression[] expressions = new OpenSeeVRMExpression[]{ - new OpenSeeVRMExpression("neutral", BlendShapePreset.Neutral, 1f, 1f, true, true, 0x70), - new OpenSeeVRMExpression("fun", BlendShapePreset.Fun, 1f, 1f, true, true, 0x71), - new OpenSeeVRMExpression("joy", BlendShapePreset.Joy, 1f, 1f, true, true, 0x72), - new OpenSeeVRMExpression("angry", BlendShapePreset.Angry, 1f, 1f, true, true, 0x73), - new OpenSeeVRMExpression("sorrow", BlendShapePreset.Sorrow, 1f, 1f, true, true, 0x74), - new OpenSeeVRMExpression("surprise", "Surprised", 1f, 1f, true, true, 0x75) + new OpenSeeVRMExpression("neutral", BlendShapePreset.Neutral, 1f, 1f, true, true, 0x70, 50f), + new OpenSeeVRMExpression("fun", BlendShapePreset.Fun, 1f, 1f, true, true, 0x71, 1f), + new OpenSeeVRMExpression("joy", BlendShapePreset.Joy, 1f, 1f, true, true, 0x72, 1f), + new OpenSeeVRMExpression("angry", BlendShapePreset.Angry, 1f, 1f, true, true, 0x73, 1f), + new OpenSeeVRMExpression("sorrow", BlendShapePreset.Sorrow, 1f, 1f, true, true, 0x74, 1f), + new OpenSeeVRMExpression("surprise", "Surprised", 1f, 1f, true, true, 0x75, 1f) }; [Tooltip("The expression configuration is initialized on startup. If it is changed and needs to be reinitialized, this can be triggered by using this flag or calling InitExpressionMap. This flag is reset to false afterwards.")] public bool reloadExpressions = false; @@ -184,10 +186,12 @@ public class OpenSeeVRMDriver : MonoBehaviour { public bool enableBlinking = true; [Tooltip("This can be set to a virtual key value. If this key is pressed together with shift and control, the expression will trigger as an override.")] public int hotkey = -1; + [Tooltip("This is the weight for this class's error when training the model.")] + public float errorWeight = 1f; [HideInInspector] public BlendShapeKey blendShapeKey; - public OpenSeeVRMExpression(string trigger, BlendShapePreset preset, float weight, float factor, bool visemes, bool blinking, int hotkey) { + public OpenSeeVRMExpression(string trigger, BlendShapePreset preset, float weight, float factor, bool visemes, bool blinking, int hotkey, float errorWeight) { this.trigger = trigger; this.blendShapePreset = preset; this.weight = weight; @@ -195,9 +199,10 @@ public class OpenSeeVRMDriver : MonoBehaviour { this.enableVisemes = visemes; this.enableBlinking = blinking; this.hotkey = hotkey; + this.errorWeight = errorWeight; } - public OpenSeeVRMExpression(string trigger, string name, float weight, float factor, bool visemes, bool blinking, int hotkey) { + public OpenSeeVRMExpression(string trigger, string name, float weight, float factor, bool visemes, bool blinking, int hotkey, float errorWeight) { this.trigger = trigger; this.customBlendShapeName = name; this.weight = weight; @@ -205,12 +210,15 @@ public class OpenSeeVRMDriver : MonoBehaviour { this.enableVisemes = visemes; this.enableBlinking = blinking; this.hotkey = hotkey; + this.errorWeight = errorWeight; } } public void InitExpressionMap() { expressionMap = new Dictionary(); + openSeeExpression.weightMap = new Dictionary(); foreach (var expression in expressions) { + openSeeExpression.weightMap.Add(expression.trigger, expression.errorWeight); if (expression.customBlendShapeName != "") expression.blendShapeKey = new BlendShapeKey(expression.customBlendShapeName); else @@ -382,10 +390,10 @@ public class OpenSeeVRMDriver : MonoBehaviour { return; } float right = 1f; - if (openSeeData.rightEyeOpen > 0.5f) + if (openSeeData.rightEyeOpen > eyeClosedThreshold) right = 0f; float left = 1f; - if (openSeeData.leftEyeOpen > 0.5f) + if (openSeeData.leftEyeOpen > eyeClosedThreshold) left = 0f; vrmBlendShapeProxy.ImmediatelySetValue(new BlendShapeKey(BlendShapePreset.Blink_R), right); vrmBlendShapeProxy.ImmediatelySetValue(new BlendShapeKey(BlendShapePreset.Blink_L), left); diff --git a/Source/SVMWrapper.cpp b/Source/SVMWrapper.cpp index 314bae2..f364d49 100644 --- a/Source/SVMWrapper.cpp +++ b/Source/SVMWrapper.cpp @@ -87,7 +87,7 @@ void destroy_problem(struct svm_problem *problem) { } extern "C" { - __declspec(dllexport) void *__cdecl trainModel(float features[], float labels[], int rows, int cols, int classes, int probability, float C) { + __declspec(dllexport) void *__cdecl trainModel(float features[], float labels[], float weights[], int rows, int cols, int classes, int probability, float C) { if (!(rows > 0 && cols > 0)) return NULL; @@ -115,10 +115,25 @@ extern "C" { sw->param.weight_label = NULL; sw->param.weight = NULL; + if (weights != NULL) { + sw->param.nr_weight = classes; + sw->param.weight_label = new int[classes]; + sw->param.weight = new double[classes]; + for (int i = 0; i < classes; i++) { + sw->param.weight_label[i] = i; + sw->param.weight[i] = (double)weights[i]; + } + } + sw->problem = new struct svm_problem(); make_problem(sw->problem, features, labels, rows, cols, sw->means, sw->sdevs); sw->model = svm_train(sw->problem, &sw->param); + if (weights != NULL) { + delete sw->param.weight_label; + delete sw->param.weight; + } + return sw; } diff --git a/Unity/OpenSeeExpression.cs b/Unity/OpenSeeExpression.cs index 6a81126..9b13028 100644 --- a/Unity/OpenSeeExpression.cs +++ b/Unity/OpenSeeExpression.cs @@ -29,6 +29,8 @@ public class OpenSeeExpression : MonoBehaviour public bool enableProbabilityTraining = false; [Tooltip("This is the SVM training C value.")] public float C = 2f; + [Tooltip("This map allows you to set weights for different classes.")] + public Dictionary weightMap = null; [Header("Toggles")] [Tooltip("When enabled, calibration data will be collected from the given OpenSee component.")] public bool recording = false; @@ -341,12 +343,13 @@ public class OpenSeeExpression : MonoBehaviour List list = new List(expressions[key]); list.RemoveAll(x => x.Length != colsFull); if (list != null && list.Count == maxSamples) { + Debug.Log("[Training info] Adding expression " + key + " to training data."); keys.Add(key); - samples += maxSamples; + samples += list.Count; } else { if (list != null && list.Count > 20) { - //Debug.Log("[Training warning] Expression " + key + " has little data and might be inaccurate."); - //accuracyWarnings.Add("Expression " + key + " has little data and might be inaccurate."); + Debug.Log("[Training warning] Expression " + key + " has little data and might be inaccurate."); + accuracyWarnings.Add("Expression " + key + " has little data and might be inaccurate."); samples += list.Count; keys.Add(key); } else { @@ -364,9 +367,19 @@ public class OpenSeeExpression : MonoBehaviour } keys.Sort(); classLabels = keys.ToArray(); + + float[] weights = null; + if (weightMap != null) { + weights = new float[classLabels.Length]; + for (int i = 0; i < classLabels.Length; i++) { + weights[i] = weightMap[classLabels[i]]; + Debug.Log("[Training info] Adding weight " + weights[i] + " for " + classLabels[i] + "."); + } + } + int train_split = maxSamples * 3 / 4; int test_split = maxSamples - train_split; - int rows_train = (classes + 2) * train_split; + int rows_train = classes * train_split; int rows_test = classes * test_split; float[] X_train = new float[rows_train * cols]; float[] X_test = new float[rows_test * cols]; @@ -389,7 +402,7 @@ public class OpenSeeExpression : MonoBehaviour } int train_split_current = train_split; if (classLabels[i] == "neutral") - train_split_current *= 3; + train_split_current *= 1; for (int j = 0; j < train_split_current; j++) { float factor = 1f; float adder = 0f; @@ -416,7 +429,7 @@ public class OpenSeeExpression : MonoBehaviour int probability = 0; if (enableProbabilityTraining) probability = 1; - model.TrainModel(X_train, y_train, i_train, cols, probability, C); + model.TrainModel(X_train, y_train, weights, i_train, cols, probability, C); confusionMatrix = model.ConfusionMatrix(X_test, y_test, i_test, out accuracy); confusionMatrixString = SVMModel.FormatMatrix(confusionMatrix, classLabels); for (int label = 0; label < classes; label++) { diff --git a/Unity/Plugins/SVMModel.cs b/Unity/Plugins/SVMModel.cs index 9b438ef..705dd2c 100644 --- a/Unity/Plugins/SVMModel.cs +++ b/Unity/Plugins/SVMModel.cs @@ -12,7 +12,7 @@ namespace OpenSee { public class SVMModel { #region DllImport [DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl)] - private static extern System.IntPtr trainModel(float[] features, float[] labels, int rows, int cols, int classes, int probability, float C); + private static extern System.IntPtr trainModel(float[] features, float[] labels, float[] weights, int rows, int cols, int classes, int probability, float C); [DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl)] private static extern void predict(System.IntPtr model, float[] features, [Out] float[] predictions, [Out] double[] probabilities, int rows); @@ -118,7 +118,7 @@ public class SVMModel { } // features is a rows*cols long float array with rows written one after another. labels is a rows long float array with the corresponding classes as integral numbers from 0 to num_classes-1, where all classes have to exist. Training on more than 10000 rows is disabled. - public bool TrainModel(float[] features, float[] labels, int rows, int cols, int probability, float C) { + public bool TrainModel(float[] features, float[] labels, float[] weights, int rows, int cols, int probability, float C) { DestroyModel(); //if (rows > 10000) // return false; @@ -143,7 +143,7 @@ public class SVMModel { maxClasses = max + 1; this.cols = cols; - model = trainModel(features, labels, rows, cols, maxClasses, probability, C); + model = trainModel(features, labels, weights, rows, cols, maxClasses, probability, C); haveModel = true; return true; diff --git a/Unity/Plugins/SVMModel.dll b/Unity/Plugins/SVMModel.dll index a273c8c..4f88e28 100644 Binary files a/Unity/Plugins/SVMModel.dll and b/Unity/Plugins/SVMModel.dll differ