copilot/OpenSeeFace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Added support for 30 point tracking to OpenSeeShowPoints and OpenSeeVRMDriver.

Browse source
This commit is contained in:
Emiliana 2020-09-02 15:31:35 +02:00
parent 082732b991
commit bde32b7213
3 changed files with 39 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

12
Examples/OpenSeeVRMDriver.cs
View file

@ -6,7 +6,7 @@ using OpenSee;
using VRM;
// This is a more comprehensive VRM avatar animator than the OpenSeeVRMExpression example.
// To use the lip sync functionality, place the OVRLipSync component somewhere in the scene.
// To use the lip sync functionality, place the OVRLipSync 1.28 component somewhere in the scene.
// No other OVR related components are required.
public class OpenSeeVRMDriver : MonoBehaviour {
@ -22,6 +22,8 @@ public class OpenSeeVRMDriver : MonoBehaviour {
public OpenSeeIKTarget openSeeIKTarget;
[Tooltip("This is the target VRM avatar's blend shape proxy.")]
public VRMBlendShapeProxy vrmBlendShapeProxy;
[Tooltip("This needs to be enabled when tracking with the fast 30 point model.")]
public bool only30Points = false;
//[Tooltip("When set to true, expression and audio data will be processed in FixedUpdate, otherwise it will be processed in Update.")]
//public bool fixedUpdate = true;
[Tooltip("When enabled, the avatar will blink automatically. Otherwise, blinks will be applied according to the face tracking's blink detection, allowing it to wink. Even for tracked eye blinks, the eye blink setting of the current expression is used.")]
@ -851,13 +853,13 @@ public class OpenSeeVRMDriver : MonoBehaviour {
toggledExpression.triggered = false;
if (!overridden && !trigger) {
if (!openSeeExpression.enabled || openSeeExpression.expressionTime + 15f < Time.time || openSeeExpression.expression == null || openSeeExpression.expression == "" || !expressionMap.ContainsKey(openSeeExpression.expression)) {
if (only30Points || !openSeeExpression.enabled || openSeeExpression.expressionTime + 15f < Time.time || openSeeExpression.expression == null || openSeeExpression.expression == "" || !expressionMap.ContainsKey(openSeeExpression.expression)) {
currentExpression = null;
} else {
currentExpression = expressionMap[openSeeExpression.expression];
currentExpression.triggered = true;
}
} else if (openSeeExpression.enabled && openSeeExpression.expressionTime + 15f > Time.time && openSeeExpression.expression != null && openSeeExpression.expression != "" && expressionMap.ContainsKey(openSeeExpression.expression)) {
} else if (!only30Points && openSeeExpression.enabled && openSeeExpression.expressionTime + 15f > Time.time && openSeeExpression.expression != null && openSeeExpression.expression != "" && expressionMap.ContainsKey(openSeeExpression.expression)) {
if (expressionMap[openSeeExpression.expression].additive)
expressionMap[openSeeExpression.expression].triggered = true;
}
@ -1024,7 +1026,7 @@ public class OpenSeeVRMDriver : MonoBehaviour {
void BlinkEyes() {
if (vrmBlendShapeProxy == null || eyeBlinker == null)
return;
if (autoBlink) {
if (autoBlink || only30Points) {
if (currentExpressionEnableBlinking) {
float blink = eyeBlinker.Blink();
vrmBlendShapeProxy.AccumulateValue(BlendShapeKey.CreateFromPreset(BlendShapePreset.Blink), blink);
@ -1416,7 +1418,7 @@ public class OpenSeeVRMDriver : MonoBehaviour {
}
FindFaceMesh();
UpdateExpression();
if (trackGaze)
if (trackGaze && !only30Points)
UpdateGaze();
BlinkEyes();
ReadAudio();

36
Unity/OpenSeeShowPoints.cs
View file

@ -7,6 +7,7 @@ namespace OpenSee {
public class OpenSeeShowPoints : MonoBehaviour {
public OpenSee openSee = null;
public int faceId = 0;
public bool only30Points = false;
public bool show3DPoints = true;
public bool applyTranslation = false;
public bool applyRotation = false;
@ -25,7 +26,10 @@ public class OpenSeeShowPoints : MonoBehaviour {
private double updated = 0.0;
private int total = 70;
private int[] lines = new int[]{/* Contour */ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, -1, /* Eyebrows */ 18, 19, 20, 21, -1, 23, 24, 25, 26, -1, 28, 29, 30, 33, 32, 33, 34, 35, -1, /* Eye */ 37, 38, 39, 40, 41, 36, /* Eye */ 43, 44, 45, 46, 47, 42, /* Mouth */ 49, 50, 51, 52, 62, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 58, 58, 62};
private int[] lines = new int[]{/* Contour */ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, -1, /* Eyebrows */ 18, 19, 20, 21, -1, 23, 24, 25, 26, -1, /* Nose */ 28, 29, 30, 33, 32, 33, 34, 35, -1, /* Eye */ 37, 38, 39, 40, 41, 36, /* Eye */ 43, 44, 45, 46, 47, 42, /* Mouth */ 49, 50, 51, 52, 62, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 58, 58, 62};
private HashSet<int> pt30Set = new HashSet<int> { 0, 2, 5, 8, 11, 14, 16, 17, 19, 21, 22, 24, 26, 27, 30, 33, 36, 37, 39, 40, 42, 43, 45, 46, 50, 55, 58, 60, 62, 64 };
private int[] linesPt30Set = new int[]{/* Contour */ 2, -1, 5, -1, -1, 8, -1, -1, 11, -1, -1, 14, -1, -1, 16, -1, -1, /* Eyebrows */ 19, -1, 21, -1, -1, 24, -1, 26, -1, -1, /* Nose */ 30, -1, -1, 33, -1, -1, -1, -1, -1, /* Eye */ 37, 39, -1, 40, 36, -1, /* Eye */ 43, 45, -1, 46, 42, -1, /* Mouth */ -1, -1, 62, -1, -1, -1, -1, 58, -1, -1, 60, -1, 62, -1, 64, -1, 58, -1, 58, 62};
void Start () {
if (openSee == null) {
@ -55,16 +59,24 @@ public class OpenSeeShowPoints : MonoBehaviour {
cylinder.transform.localPosition = new Vector3(0f, 0f, -4f);
cylinder.transform.localScale = new Vector3(1f, 4f, 1f);
}
if (only30Points && i < 66 && !pt30Set.Contains(i))
gameObjects[i].SetActive(false);
}
for (int i = 0; i < 68; i++) {
if (i == 66) {
GameObject lineGameObject = new GameObject("LineGameObject");
lineGameObject.transform.SetParent(gameObjects[48].transform);
if (only30Points)
lineGameObject.transform.SetParent(gameObjects[50].transform);
else
lineGameObject.transform.SetParent(gameObjects[48].transform);
lineGameObject.layer = gameObject.layer;
lineRenderers[i] = lineGameObject.AddComponent(typeof(LineRenderer)) as LineRenderer;
} else if (i == 67) {
GameObject lineGameObject = new GameObject("LineGameObject");
lineGameObject.transform.SetParent(gameObjects[53].transform);
if (only30Points)
lineGameObject.transform.SetParent(gameObjects[55].transform);
else
lineGameObject.transform.SetParent(gameObjects[53].transform);
lineGameObject.layer = gameObject.layer;
lineRenderers[i] = lineGameObject.AddComponent(typeof(LineRenderer)) as LineRenderer;
}
@ -172,17 +184,25 @@ public class OpenSeeShowPoints : MonoBehaviour {
}
}
for (int i = 0; i < 68; i++) {
if (lines[i] == -1)
if ((!only30Points && lines[i] == -1) || (only30Points && linesPt30Set[i] == -1))
continue;
if (!showLines || lineMaterial == null) {
lineRenderers[i].enabled = false;
} else {
int a = i;
int b = lines[i];
if (i == 66)
a = 48;
if (i == 67)
a = 53;
if (only30Points) {
b = linesPt30Set[i];
if (i == 66)
a = 50;
if (i == 67)
a = 55;
} else {
if (i == 66)
a = 48;
if (i == 67)
a = 53;
}
Color color = Color.Lerp(Color.red, Color.green, Mathf.Lerp(0.5f, openSeeData.confidence[a], openSeeData.confidence[b]));
lineRenderers[i].enabled = true;
lineRenderers[i].widthMultiplier = lineWidth;

4
tracker.py
View file

@ -731,6 +731,10 @@ class Tracker():
lms[np.isnan(lms).any(axis=1)] = np.array([0.,0.,0.], dtype=np.float32)
if self.model_type < 0:
lms = lms[[0,0,1,1,1,2,2,2,3,3,3,4,4,4,5,5,6,7,7,8,8,9,10,10,11,11,12,21,21,21,22,23,23,23,23,23,13,14,14,15,16,16,17,18,18,19,20,20,24,25,25,25,26,26,27,27,27,24,24,28,28,28,26,29,29,29]]
#lms[[1,3,4,6,7,9,10,12,13,15,18,20,23,25,38,40,44,46]] += lms[[2,2,5,5,8,8,11,11,14,16,19,21,24,26,39,39,45,45]]
#lms[[3,4,6,7,9,10,12,13]] += lms[[5,5,8,8,11,11,14,14]]
#lms[[1,15,18,20,23,25,38,40,44,46]] /= 2.0
#lms[[3,4,6,7,9,10,12,13]] /= 3.0
part_avg = np.mean(np.partition(lms[:,2],3)[0:3])
if part_avg < 0.65:
avg_conf = part_avg