copilot/OpenSeeFace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Added hybrid lip sync/mouth tracking to OpenSeeVRMDriver.

Browse source
This commit is contained in:
Emiliana 2020-06-27 13:13:45 +02:00
parent 8d2df41d24
commit baf4d546c0
1 changed files with 30 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

31
Examples/OpenSeeVRMDriver.cs
View file

@ -83,6 +83,8 @@ public class OpenSeeVRMDriver : MonoBehaviour {
[Header("Lip sync settings")]
[Tooltip("This allows you to enable and disable lip sync. When disabled, mouth tracking is used instead.")]
public bool lipSync = true;
[Tooltip("When enabled, camera based mouth tracking will be used when no viseme is detected.")]
public bool hybridLipSync = false;
[Tooltip("This is the mouth tracking smoothing factor, with 0 being no smoothing and 1 being a fixed mouth.")]
[Range(0, 1)]
public float mouthSmoothing = 0.6f;
@ -162,6 +164,10 @@ public class OpenSeeVRMDriver : MonoBehaviour {
private int mouthInterpolationCount = 0;
private int mouthInterpolationState = 0;
private double startedSilVisemes = -10;
private double silVisemeHybridThreshold = 0.3;
private bool wasSilViseme = true;
private double lastBlink = 0f;
private float lastBlinkLeft = 0f;
private float lastBlinkRight = 0f;
@ -242,6 +248,23 @@ public class OpenSeeVRMDriver : MonoBehaviour {
}
float weight;
OVRLipSync.Viseme current = GetActiveViseme(out weight);
if (current == OVRLipSync.Viseme.sil && weight > 0.9999f) {
if (wasSilViseme) {
if (hybridLipSync && Time.time - startedSilVisemes > silVisemeHybridThreshold) {
ApplyMouthShape();
return;
}
} else {
startedSilVisemes = Time.time;
}
wasSilViseme = true;
} else {
if (wasSilViseme)
startedSilVisemes = Time.time;
wasSilViseme = false;
if (hybridLipSync && Time.time - startedSilVisemes < silVisemeHybridThreshold)
ApplyMouthShape(true);
}
weight = Mathf.Clamp(weight * 1.5f, 0f, 1f);
float[] values = catsData[current];
for (int i = 0; i < 5; i++) {
@ -258,7 +281,7 @@ public class OpenSeeVRMDriver : MonoBehaviour {
}
}
void ApplyMouthShape() {
void ApplyMouthShape(bool fadeOnly) {
if (vrmBlendShapeProxy == null)
return;
if (visemePresetMap == null)
@ -280,6 +303,8 @@ public class OpenSeeVRMDriver : MonoBehaviour {
}
do {
if (fadeOnly)
break;
if (mouthUseSquelch && audioVolume < mouthSquelch)
break;
if (open < stabilizer && Mathf.Abs(wide) < stabilizer)
@ -354,6 +379,10 @@ public class OpenSeeVRMDriver : MonoBehaviour {
}
}
void ApplyMouthShape() {
ApplyMouthShape(false);
}
void UpdateBrows() {
if ((browClips == null && (faceMesh == null || faceType < 0)) || openSeeData == null) {
return;