This adds the gaze tracker.

Examples/OpenSeeVRMDriver.cs

@@ -12,12 +12,25 @@ public class OpenSeeVRMDriver : MonoBehaviour {
     [Header("Settings")]
     [Tooltip("This is the OpenSeeExpression module used for expression prediction.")]
     public OpenSeeExpression openSeeExpression;
+    [Tooltip("This is the OpenSeeIKTarget module used to move the avatar.")]
+    public OpenSeeIKTarget openSeeIKTarget;
     [Tooltip("This is the target VRM avatar's blend shape proxy.")]
     public VRMBlendShapeProxy vrmBlendShapeProxy;
-    [Tooltip("When set to true, expression and audio data will be processed in FixedUpdate, otherwise it will be processed in Update.")]
+    //[Tooltip("When set to true, expression and audio data will be processed in FixedUpdate, otherwise it will be processed in Update.")]
-    public bool fixedUpdate = true;
+    //public bool fixedUpdate = true;
     [Tooltip("When enabled, the avatar will blink automatically. Otherwise, blinks will be applied according to the face tracking's blink detection, allowing it to wink. Even for tracked eye blinks, the eye blink setting of the current expression is used.")]
     public bool autoBlink = true;
+    [Tooltip("When enabled, the avatar's eye will move according to the face tracker's gaze tracking.")]
+    public bool gazeTracking = true;
+    [Tooltip("This is the right eye bone. When either eye bone is not set, the VRM look direction blendshapes are used instead.")]
+    public Transform rightEye;
+    [Tooltip("This is the left eye bone. When either eye bone is not set, the VRM look direction blendshapes are used instead.")]
+    public Transform leftEye;
+    [Tooltip("This is the gaze smoothing factor, with 0 being no smoothing and 1 being a fixed gaze.")]
+    [Range(0, 1)]
+    public float gazeSmoothing = 0.9f;
+    [Tooltip("For bone based gaze tracking, the conversion factor from [-1, 1] to degrees should be entered here. For blendshapes, sometimes gaze tracking does not follow the gaze strongly enough. With these factors, its effect can be strengthened.")]
+    public Vector2 gazeFactor = new Vector2(5f, 5f);
     [Tooltip("This component lets you customize the automatic eye blinking.")]
     public OpenSeeEyeBlink eyeBlinker = new OpenSeeEyeBlink();
     [Tooltip("This component lets configure your VRM expressions.")]
@@ -39,7 +52,7 @@ public class OpenSeeVRMDriver : MonoBehaviour {
     public AudioSource audioSource = null;
     [Tooltip("This is the microphone audio will be captured from. A list can be retrieved from Microphone.devices.")]
     public string mic = null;
-    [Tooltip("When enabled, the lip sync function will be initialized or reinitialized in the next Update or FixedUpdate call, according to the fixedUpdate flag. This flag is reset to false afterwards. It is also possible to call InitializeLipSync instead.")]
+    [Tooltip("When enabled, the lip sync function will be initialized or reinitialized in the next Update or FixedUpdate call, according to the IK target's fixedUpdate flag. This flag is reset to false afterwards. It is also possible to call InitializeLipSync instead.")]
     public bool initializeLipSync = false;
     [Range(1, 100)]
     [Tooltip("This sets the viseme smoothing in a range from 1 to 100, where 1 means no smoothing and values above 90 mean pretty much no visemes.")]
@@ -69,6 +82,14 @@ public class OpenSeeVRMDriver : MonoBehaviour {
     private BlendShapeKey[] visemePresetMap;
     private Dictionary<string, OpenSeeVRMExpression> expressionMap;
     private OpenSeeVRMExpression currentExpression = null;
+    private OpenSee.OpenSee.OpenSeeData openSeeData = null;
+    private double lastGaze = 0f;
+    private float lastLookUpDown = 0f;
+    private float lastLookLeftRight = 0f;
+    private float currentLookUpDown = 0f;
+    private float currentLookLeftRight = 0f;
+    private int interpolationCount = 0;
+    private int interpolationState = 0;
 
     private Dictionary<OVRLipSync.Viseme, float[]> catsData = null;
     void InitCatsData() {
@@ -192,6 +213,93 @@ public class OpenSeeVRMDriver : MonoBehaviour {
         vrmBlendShapeProxy.ImmediatelySetValue(currentExpression.blendShapeKey, currentExpression.weight);
     }
     
+    void GetLookParameters(ref float lookLeftRight, ref float lookUpDown, bool update, int gazePoint, int topRight, int topLeft, int bottomRight, int bottomLeft) {
+        float borderRight = (openSeeData.points3D[topRight].x + openSeeData.points3D[bottomRight].x) / 2.0f;
+        float borderLeft = (openSeeData.points3D[topLeft].x + openSeeData.points3D[bottomLeft].x) / 2.0f;
+        float horizontalCenter = (borderRight + borderLeft) / 2.0f;
+        float horizontalRadius = (Mathf.Abs(horizontalCenter - borderRight) + Mathf.Abs(borderLeft - horizontalCenter)) / 2.0f;
+        float borderTop = (openSeeData.points3D[topRight].y + openSeeData.points3D[topLeft].y) / 2.0f;
+        float borderBottom = (openSeeData.points3D[bottomRight].y + openSeeData.points3D[bottomLeft].y) / 2.0f;
+        float verticalCenter = (borderTop + borderBottom) / 2.0f;
+        float verticalRadius = (Mathf.Abs(verticalCenter - borderTop) + Mathf.Abs(borderBottom - verticalCenter)) / 2.0f;
+        float x = openSeeData.points3D[gazePoint].x;
+        float y = openSeeData.points3D[gazePoint].y;
+        if (!update) {
+            lookLeftRight = Mathf.Clamp(gazeFactor.x * (x - horizontalCenter) / horizontalRadius, -1f, 1f);
+            lookUpDown = Mathf.Clamp(gazeFactor.y * (y - verticalCenter) / verticalRadius, -1f, 1f);
+        } else {
+            lookLeftRight += Mathf.Clamp(gazeFactor.x * (x - horizontalCenter) / horizontalRadius, -1f, 1f);
+            lookLeftRight /= 2.0f;
+            lookUpDown += Mathf.Clamp(gazeFactor.y * (y - verticalCenter) / verticalRadius, -1f, 1f);
+            lookUpDown /= 2.0f;
+        }
+    }
+    
+    void UpdateGaze() {
+        if (openSeeData == null || vrmBlendShapeProxy == null || openSeeIKTarget == null || !(openSeeIKTarget.averageInterpolations > 0f))
+            return;
+        
+        float lookUpDown = 0f;
+        float lookLeftRight = 0f;
+
+        if (lastGaze < openSeeData.time) {
+            GetLookParameters(ref lookLeftRight, ref lookUpDown, false, 66, 37, 38, 41, 40);
+            GetLookParameters(ref lookLeftRight, ref lookUpDown, true,  67, 43, 44, 47, 46);
+            
+            lookLeftRight = Mathf.Lerp(lastLookLeftRight, lookLeftRight, gazeSmoothing);
+            lookUpDown = Mathf.Lerp(lastLookUpDown, lookUpDown, gazeSmoothing);
+            lastLookLeftRight = currentLookLeftRight;
+            lastLookUpDown = currentLookUpDown;
+            currentLookLeftRight = lookLeftRight;
+            currentLookUpDown = lookUpDown;
+            lastGaze = openSeeData.time;
+            if (interpolationState < 2)
+                interpolationState++;
+            interpolationCount = 0;
+        }
+        
+        if (interpolationState < 2)
+            return;
+        
+        float t = Mathf.Clamp((float)interpolationCount / openSeeIKTarget.averageInterpolations, 0f, 0.985f);
+        lookLeftRight = Mathf.Lerp(lastLookLeftRight, currentLookLeftRight, t);
+        lookUpDown = Mathf.Lerp(lastLookUpDown, currentLookUpDown, t);
+        interpolationCount++;
+        
+        if (!openSeeIKTarget.mirrorMotion)
+            lookLeftRight = -lookLeftRight;
+        
+        if (leftEye == null || rightEye == null) {
+            vrmBlendShapeProxy.ImmediatelySetValue(new BlendShapeKey(BlendShapePreset.LookUp), 0f);
+            vrmBlendShapeProxy.ImmediatelySetValue(new BlendShapeKey(BlendShapePreset.LookDown), 0f);
+            vrmBlendShapeProxy.ImmediatelySetValue(new BlendShapeKey(BlendShapePreset.LookLeft), 0);
+            vrmBlendShapeProxy.ImmediatelySetValue(new BlendShapeKey(BlendShapePreset.LookRight), 0f);
+            if (gazeTracking) {
+                if (lookUpDown > 0f) {
+                    vrmBlendShapeProxy.ImmediatelySetValue(new BlendShapeKey(BlendShapePreset.LookUp), lookUpDown);
+                } else {
+                    vrmBlendShapeProxy.ImmediatelySetValue(new BlendShapeKey(BlendShapePreset.LookDown), -lookUpDown);
+                }
+                
+                if (lookLeftRight > 0f) {
+                    vrmBlendShapeProxy.ImmediatelySetValue(new BlendShapeKey(BlendShapePreset.LookLeft), lookLeftRight);
+                } else {
+                    vrmBlendShapeProxy.ImmediatelySetValue(new BlendShapeKey(BlendShapePreset.LookRight), -lookLeftRight);
+                }
+            }
+        } else {
+            //vrmLookAtHead.UpdateType = VRM.UpdateType.None;
+            //vrmLookAtHead.RaiseYawPitchChanged(gazeFactor.y * lookLeftRight, gazeFactor.x * lookUpDown);
+            rightEye.localRotation = Quaternion.identity;
+            leftEye.localRotation = Quaternion.identity;
+            if (gazeTracking) {
+                Quaternion rotation = Quaternion.AngleAxis(gazeFactor.x * lookUpDown, Vector3.right) * Quaternion.AngleAxis(gazeFactor.y * lookLeftRight, Vector3.up);
+                rightEye.localRotation = rotation;
+                leftEye.localRotation = rotation;
+            }
+        }
+    }
+
     void BlinkEyes() {
         if (vrmBlendShapeProxy == null || eyeBlinker == null)
             return;
@@ -204,7 +312,6 @@ public class OpenSeeVRMDriver : MonoBehaviour {
                 vrmBlendShapeProxy.ImmediatelySetValue(new BlendShapeKey(BlendShapePreset.Blink_L), 0f);
             }
         } else if (openSeeExpression != null && openSeeExpression.openSee != null) {
-            OpenSee.OpenSee.OpenSeeData openSeeData = openSeeExpression.openSee.GetOpenSeeData(openSeeExpression.faceId);
             if (openSeeData == null || !currentExpression.enableBlinking) {
                 vrmBlendShapeProxy.ImmediatelySetValue(new BlendShapeKey(BlendShapePreset.Blink_R), 0f);
                 vrmBlendShapeProxy.ImmediatelySetValue(new BlendShapeKey(BlendShapePreset.Blink_L), 0f);
@@ -444,18 +551,21 @@ public class OpenSeeVRMDriver : MonoBehaviour {
     }
 
     void RunUpdates() {
+        if (openSeeExpression != null && openSeeExpression.openSee != null)
+            openSeeData = openSeeExpression.openSee.GetOpenSeeData(openSeeExpression.faceId);
         if (initializeLipSync) {
             InitializeLipSync();
             initializeLipSync = false;
         }
         UpdateExpression();
+        UpdateGaze();
         BlinkEyes();
         ReadAudio();
         ApplyVisemes();
     }
 
     void FixedUpdate() {
-        if (fixedUpdate) {
+        if (openSeeIKTarget.fixedUpdate) {
             RunUpdates();
         }
     }
@@ -465,7 +575,7 @@ public class OpenSeeVRMDriver : MonoBehaviour {
             OVRLipSync.SendSignal(context, OVRLipSync.Signals.VisemeSmoothing, smoothAmount, 0);
             lastSmoothing = smoothAmount;
         }
-        if (!fixedUpdate) {
+        if (!openSeeIKTarget.fixedUpdate) {
             RunUpdates();
         }
     }

Unity/OpenSee.cs

@@ -15,7 +15,7 @@ public class OpenSee : MonoBehaviour {
     [Tooltip("This is the port the server will listen for tracking packets on")]
     public int listenPort = 11573;
 
-    private const int nPoints = 66;
+    private const int nPoints = 68;
     private const int packetFrameSize = 8 + 4 + 2 * 4 + 2 * 4 + 1 + 4 + 3 * 4 + 3 * 4 + 4 * 4 + 4 * 68 + 4 * 2 * 68 + 4 * 3 * 70;
 
     [Header("Tracking data")]
@@ -36,6 +36,10 @@ public class OpenSee : MonoBehaviour {
         public float rightEyeOpen;
         [Tooltip("This field tells you how likely it is that the left eye is open.")]
         public float leftEyeOpen;
+        [Tooltip("This field tells you how likely it is that the right eye is open.")]
+        public Quaternion rightGaze;
+        [Tooltip("This field tells you how likely it is that the left eye is open.")]
+        public Quaternion leftGaze;
         [Tooltip("This field tells you if 3D points have been successfully estimated from the 2D points. If this is false, do not rely on pose or 3D data.")]
         public bool got3DPoints;
         [Tooltip("This field contains the error for fitting the original 3D points. It shouldn't matter much, but it it is very high, something is probably wrong")]
@@ -58,7 +62,12 @@ public class OpenSee : MonoBehaviour {
         public OpenSeeData() {
             confidence = new float[nPoints];
             points = new Vector2[nPoints];
-            points3D = new Vector3[nPoints];
+            points3D = new Vector3[nPoints + 2];
+        }
+        
+        private Vector3 swapX(Vector3 v) {
+            v.x = -v.x;
+            return v;
         }
 
         private float readFloat(byte[] b, ref int o) {
@@ -120,22 +129,16 @@ public class OpenSee : MonoBehaviour {
                 confidence[i] = readFloat(b, ref o);
             }
 
-            // Gaze tracker results are still bad, ignore them
-            readFloat(b, ref o); readFloat(b, ref o);
-
             for (int i = 0; i < nPoints; i++) {
                 points[i] = readVector2(b, ref o);
             }
 
-            // Gaze tracker results are still bad, ignore them
+            for (int i = 0; i < nPoints + 2; i++) {
-            readVector2(b, ref o); readVector2(b, ref o);
-
-            for (int i = 0; i < nPoints; i++) {
                 points3D[i] = readVector3(b, ref o);
             }
             
-            // Gaze tracker results are still bad, ignore them
+            rightGaze = Quaternion.LookRotation(swapX(points3D[66]) - swapX(points3D[68])) * Quaternion.AngleAxis(180, Vector3.right) * Quaternion.AngleAxis(180, Vector3.forward);
-            readVector3(b, ref o); readVector3(b, ref o); readVector3(b, ref o); readVector3(b, ref o);
+            leftGaze = Quaternion.LookRotation(swapX(points3D[67]) - swapX(points3D[69])) * Quaternion.AngleAxis(180, Vector3.right) * Quaternion.AngleAxis(180, Vector3.forward);
         }
     }
 

Unity/OpenSeeShowPoints.cs

@@ -21,12 +21,19 @@ public class OpenSeeShowPoints : MonoBehaviour {
         if (openSee == null) {
             openSee = GetComponent<OpenSee>();
         }
-        gameObjects = new GameObject[68];
+        gameObjects = new GameObject[70];
-        for (int i = 0; i < 68; i++) {
+        for (int i = 0; i < 70; i++) {
             gameObjects[i] = GameObject.CreatePrimitive(PrimitiveType.Sphere);
             gameObjects[i].name = "Point " + (i + 1);
             gameObjects[i].transform.SetParent(transform);
             gameObjects[i].transform.localScale = new Vector3(0.025f, 0.025f, 0.025f);
+            if (i >= 68) {
+                GameObject cylinder = GameObject.CreatePrimitive(PrimitiveType.Cylinder);
+                cylinder.transform.SetParent(gameObjects[i].transform);
+                cylinder.transform.localEulerAngles = new Vector3(90f, 0f, 0f);
+                cylinder.transform.localPosition = new Vector3(0f, 0f, -4f);
+                cylinder.transform.localScale = new Vector3(1f, 4f, 1f);
+            }
         }
         centerBall = GameObject.CreatePrimitive(PrimitiveType.Sphere);
         centerBall.name = "Center";
@@ -50,13 +57,20 @@ public class OpenSeeShowPoints : MonoBehaviour {
         }
         if (show3DPoints) {
             centerBall.gameObject.SetActive(false);
-            for (int i = 0; i < 66; i++) {
+            for (int i = 0; i < 70; i++) {
-                if (openSeeData.got3DPoints && openSeeData.confidence[i] > minConfidence) {
+                if (openSeeData.got3DPoints && (i >= 68 || openSeeData.confidence[i] > minConfidence)) {
                     Renderer renderer = gameObjects[i].GetComponent<Renderer>();
-                    renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i]));
                     Vector3 pt = openSeeData.points3D[i];
                     pt.x = -pt.x;
                     gameObjects[i].transform.localPosition = pt;
+                    if (i < 68)
+                        renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i]));
+                    else {
+                        if (i == 68)
+                            gameObjects[i].transform.localRotation = openSeeData.rightGaze;
+                        else
+                            gameObjects[i].transform.localRotation = openSeeData.leftGaze;
+                    }
                 } else {
                     Renderer renderer = gameObjects[i].GetComponent<Renderer>();
                     renderer.material.SetColor("_Color", Color.cyan);

facetracker.py

@@ -120,8 +120,8 @@ try:
         for face_num, f in enumerate(faces):
             f = copy.copy(f)
             f.id += args.face_id_offset
-            right_state = "O" if f.eye_blink[0] > 0.5 else "-"
+            right_state = "O" if f.eye_blink[0] > 0.8 else "-"
-            left_state = "O" if f.eye_blink[1] > 0.5 else "-"
+            left_state = "O" if f.eye_blink[1] > 0.8 else "-"
             if args.silent == 0:
                 print(f"Confidence[{f.id}]: {f.conf:.4f} / 3D fitting error: {f.pnp_error:.4f} / Eyes: {left_state}, {right_state}")
             detected = True
@@ -154,9 +154,9 @@ try:
                 packet.extend(bytearray(struct.pack("f", x)))
                 if not log is None:
                     log.write(f",{y},{x},{c}")
-                if pt_num == 66:# and right_open < 0.5:
+                if pt_num == 66 and f.eye_blink[0] < 0.5:
                     continue
-                if pt_num == 67:# and left_open < 0.5:
+                if pt_num == 67 and f.eye_blink[0] < 0.5:
                     continue
                 x = int(x + 0.5)
                 y = int(y + 0.5)
@@ -165,17 +165,20 @@ try:
                         frame = cv2.putText(frame, str(f.id), (int(f.bbox[0]), int(f.bbox[1])), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (255,0,255))
                     if args.visualize > 2:
                         frame = cv2.putText(frame, str(pt_num), (int(y), int(x)), cv2.FONT_HERSHEY_SIMPLEX, 0.25, (255,255,0))
+                    color = (0, 0, 255)
+                    if pt_num >= 66:
+                        color = (255, 255, 0)
                     if not (x < 0 or y < 0 or x >= height or y >= width):
-                        frame[int(x), int(y)] = (0, 0, 255)
+                        frame[int(x), int(y)] = color
                     x += 1
                     if not (x < 0 or y < 0 or x >= height or y >= width):
-                        frame[int(x), int(y)] = (0, 0, 255)
+                        frame[int(x), int(y)] = color
                     y += 1
                     if not (x < 0 or y < 0 or x >= height or y >= width):
-                        frame[int(x), int(y)] = (0, 0, 255)
+                        frame[int(x), int(y)] = color
                     x -= 1
                     if not (x < 0 or y < 0 or x >= height or y >= width):
-                        frame[int(x), int(y)] = (0, 0, 255)
+                        frame[int(x), int(y)] = color
             if args.pnp_points != 0 and (args.visualize != 0 or not out is None):
                 if args.pnp_points > 1:
                     projected = cv2.projectPoints(f.face_3d, f.rotation, f.translation, tracker.camera, tracker.dist_coeffs)

tracker.py

@@ -25,6 +25,32 @@ def clamp_to_im(pt, w, h):
         y = h-1
     return (int(x), int(y+1))
 
+def rotate(origin, point, angle):
+    angle = -angle
+    ox, oy = origin
+    px, py = point
+
+    qx = ox + math.cos(angle) * (px - ox) - math.sin(angle) * (py - oy)
+    qy = oy + math.sin(angle) * (px - ox) + math.cos(angle) * (py - oy)
+    return qx, qy
+
+def angle(p1, p2):
+    p1 = np.array(p1)
+    p2 = np.array(p2)
+    a = np.arctan2(*(p2 - p1)[::-1])
+    return (a % (2 * np.pi))
+
+def compensate(p1, p2):
+    a = angle(p1, p2)
+    return rotate(p1, p2, a), a
+
+def rotate_image(image, angle, center):
+    (h, w) = image.shape[:2]
+    angle = np.rad2deg(angle)
+    M = cv2.getRotationMatrix2D((center[0], center[1]), angle, 1.0)
+    rotated = cv2.warpAffine(image, M, (w, h))
+    return rotated
+
 def intersects(r1, r2):
     r1_x1, r1_y1, r1_x2, r1_y2 = r1
     r1_x2 += r1_x1
@@ -93,10 +119,15 @@ class FaceInfo():
         self.id = id
         self.frame_count = -1
         self.tracker = tracker
+        self.face_3d = copy.copy(self.tracker.face_3d)
         self.reset()
         self.alive = False
         self.coord = None
 
+        self.limit_3d_adjustment = True
+        self.update_count_delta = 50.
+        self.update_count_max = 1500.
+
     def reset(self):
         self.alive = False
         self.conf = None
@@ -111,8 +142,8 @@ class FaceInfo():
         self.pts_3d = None
         self.eye_blink = None
         self.bbox = None
-        self.face_3d = copy.copy(self.tracker.face_3d)
         self.contour = np.zeros((21,3))
+        self.update_counts = np.zeros((66,2))
         self.update_contour()
 
     def update(self, result, coord, frame_count):
@@ -133,19 +164,19 @@ class FaceInfo():
 
     def adjust_3d(self):
         r = 1.0 + np.random.random_sample((66,3)) * 0.02 - 0.01
+        update_type = -1
         if self.euler[0] > -165 and self.euler[0] < 145:
             r[:, :] = 1.0
-            print("Skipping all")
         elif self.euler[1] > -10 and self.euler[1] < 20:
             r[:, 2] = 1.0
-            print("Skipping depth")
+            update_type = 0
         else:
             r[:, 0:2] = 1.0
             if self.euler[2] > 120 or self.euler[2] < 60:
                 r[:, 2] = 1.0
-                print("Skipping all-b")
             # Enable only one side of the points, depending on direction
             elif self.euler[1] < -10:
+                update_type = 1
                 r[0:8, 2] = 1.0
                 r[17:22, 2] = 1.0
                 r[31:33, 2] = 1.0
@@ -153,6 +184,7 @@ class FaceInfo():
                 for i in [48, 49, 56, 57, 58, 59, 65]:
                     r[i, 2] = 1.0
             else:
+                update_type = 1
                 r[8:17, 2] = 1.0
                 r[22:27, 2] = 1.0
                 r[34:36, 2] = 1.0
@@ -168,10 +200,28 @@ class FaceInfo():
         for i in range(66):
             lm = self.lms[i, 0:2]
             if np.linalg.norm(c_projected[i] - lm) < np.linalg.norm(o_projected[i] - lm):
-                updated[i] = c[i]
+                can_update = False
-                changed = True
+                if not self.limit_3d_adjustment or self.update_counts[i, update_type] < self.update_counts[i, abs(update_type - 1)] + self.update_count_delta:
-        self.face_3d = self.face_3d * 0.3 + updated * 0.7
+                    can_update = True
+                    self.update_counts[i, update_type] += 1
+                if can_update:
+                    updated[i] = c[i]
+                    changed = True
+
         if changed:
+            # Update weighted by point confidence
+            weights = np.zeros((66,3))
+            weights[:, :] = self.lms[0:66, 2:3]
+            weights[weights > 0.7] = 1.0
+            weights = 1.0 - weights
+            if self.limit_3d_adjustment:
+                # And by update count
+                weights[:, 0:2] *= self.update_counts[:, 0:1] / self.update_count_max
+                weights[:, 2] *= self.update_counts[:, 1] / self.update_count_max
+                weights = 1 - 1 / (1 + np.power(1 / (weights / self.update_count_max) - 1, -3))
+                weights[weights > 0.975] = 0.975
+            self.face_3d[0:66] = self.face_3d[0:66] * weights + updated[0:66] * (1. - weights)
+
             self.update_contour()
 
 
@@ -224,7 +274,7 @@ class Tracker():
         options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
         options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
         options.log_severity_level = 3
-        self.gaze_model = onnxruntime.InferenceSession(os.path.join(model_base_path, "gaze_opt.onnx"), sess_options=options)
+        self.gaze_model = onnxruntime.InferenceSession(os.path.join(model_base_path, "mnv3_gaze32_16_split_opt.onnx"), sess_options=options)
 
         self.faceCascade = cv2.CascadeClassifier()
         self.faceCascade.load(os.path.join(model_base_path, "haarcascade_frontalface_alt2.xml"))
@@ -332,6 +382,7 @@ class Tracker():
         self.res = 224. if self.model_type != 0 else 112.
         self.res_i = int(self.res)
         self.no_gaze = no_gaze
+        self.debug_gaze = False
         self.face_info = [FaceInfo(id, self) for id in range(max_faces)]
 
     def landmarks(self, tensor, crop_info):
@@ -389,15 +440,32 @@ class Tracker():
         pnp_error = 0.0
         for i, pt in enumerate(face_info.face_3d):
             if i == 68:
-                eye_center = np.transpose(pts_3d[36:42,0:2], (1,0)).mean(1)
+                # Right eyeball
-                pt_3d = np.array([eye_center[0], eye_center[1], face_info.face_3d[i][2]], np.float32)
+                # Eyeballs have an average diameter of 12.5mm and and the distance between eye corners is 30-35mm, so a conversion factor of 0.385 can be applied
+                eye_center = (pts_3d[36] + pts_3d[39]) / 2.0
+                d_corner = np.linalg.norm(pts_3d[36] - pts_3d[39])
+                depth = 0.385 * d_corner
+                pt_3d = np.array([eye_center[0], eye_center[1], eye_center[2] - depth])
                 pts_3d[i] = pt_3d
                 continue
             if i == 69:
-                eye_center = np.transpose(pts_3d[42:48,0:2], (1,0)).mean(1)
+                # Left eyeball
-                pt_3d = np.array([eye_center[0], eye_center[1], face_info.face_3d[i][2]], np.float32)
+                eye_center = (pts_3d[42] + pts_3d[45]) / 2.0
+                d_corner = np.linalg.norm(pts_3d[42] - pts_3d[45])
+                depth = 0.385 * d_corner
+                pt_3d = np.array([eye_center[0], eye_center[1], eye_center[2] - depth])
                 pts_3d[i] = pt_3d
                 continue
+            if i == 66:
+                d1 = np.linalg.norm(lms[i,0:2] - lms[36,0:2])
+                d2 = np.linalg.norm(lms[i,0:2] - lms[39,0:2])
+                d = d1 + d2
+                pt = (pts_3d[36] * d1 + pts_3d[39] * d2) / d
+            if i == 67:
+                d1 = np.linalg.norm(lms[i,0:2] - lms[42,0:2])
+                d2 = np.linalg.norm(lms[i,0:2] - lms[45,0:2])
+                d = d1 + d2
+                pt = (pts_3d[42] * d1 + pts_3d[45] * d2) / d
             reference = rmat.dot(pt)
             reference = reference + face_info.translation
             reference = self.camera.dot(reference)
@@ -429,24 +497,56 @@ class Tracker():
             im = np.expand_dims(im, 1)
         return im
 
-    def prepare_eye(self, frame, lms):
+    def corners_to_eye(self, corners, w, h, flip):
-        lms = np.transpose(lms, (1,0))
+        ((cx1, cy1), (cx2, cy2)) = corners
-        center = lms.mean(1)
+        c1 = np.array([cx1, cy1])
-        radius = np.abs(lms - np.expand_dims(center, 1)).max(1).max() * 0.8
+        c2 = np.array([cx2, cy2])
-        x = center[1]
+        c2, angle = compensate(c1, c2)
-        y = center[0]
+        center = (c1 + c2) / 2.0
-        x1, y1 = clamp_to_im((x - radius, y - radius), self.width, self.height)
+        radius = np.linalg.norm(c1 - c2) / 2.0
-        x2, y2 = clamp_to_im((x + radius, y + radius), self.width, self.height)
+        radius = np.array([radius * 1.4, radius * 1.2])
-        dist = min(x2-x1, y2-y1)
+        upper_left = clamp_to_im(center - radius, w, h)
-        x2 = x1 + dist
+        lower_right = clamp_to_im(center + radius, w, h)
-        y2 = y1 + dist
+        return upper_left, lower_right, center, radius, c1, angle
-        im = np.float32(frame[int(y1):int(y2), int(x1):int(x2),::-1])
+
+    def prepare_eye(self, frame, full_frame, lms, flip):
+        outer_pt = tuple(lms[0])
+        inner_pt = tuple(lms[1])
+        h, w, _ = frame.shape
+        (x1, y1), (x2, y2), center, radius, reference, angle = self.corners_to_eye((outer_pt, inner_pt), w, h, flip)
+        im = rotate_image(frame[:, :, ::], angle, reference)
+        im = im[int(y1):int(y2), int(x1):int(x2),:]
         if np.prod(im.shape) < 1:
-            return None, None, None, None
+            return None, None, None, None, None, None
-        im = cv2.resize(im, (32, 32), interpolation=cv2.INTER_LINEAR) / self.std - self.mean
+        if flip:
+            im = cv2.flip(im, 1)
+        scale = np.array([(x2 - x1), (y2 - y1)]) / 32.
+        im = cv2.resize(im, (32, 32), interpolation=cv2.INTER_LINEAR)
+        if self.debug_gaze:
+            if not flip:
+                full_frame[0:32, 0:32] = im
+            else:
+                full_frame[0:32, 32:64] = im
+        im = im.astype(np.float32)[:,:,::-1] / self.std - self.mean
         im = np.expand_dims(im, 0)
-        im = np.transpose(im, (0,3,1,2))
+        im = np.transpose(im, (0,3,2,1))
-        return (im, x1, y1, dist / 32.)
+        return im, x1, y1, scale, reference, angle
+
+    def extract_face(self, frame, lms):
+        lms = np.array(lms)[:,0:2][:,::-1]
+        x1, y1 = tuple(lms.min(0))
+        x2, y2 = tuple(lms.max(0))
+        radius_x = 1.2 * (x2 - x1) / 2.0
+        radius_y = 1.2 * (y2 - y1) / 2.0
+        radius = np.array((radius_x, radius_y))
+        center = (np.array((x1, y1)) + np.array((x2, y2))) / 2.0
+        w, h, _ = frame.shape
+        x1, y1 = clamp_to_im(center - radius, h, w)
+        x2, y2 = clamp_to_im(center + radius + 1, h, w)
+        offset = np.array((x1, y1))
+        lms = (lms[:, 0:2] - offset).astype(np.int)
+        frame = frame[y1:y2, x1:x2]
+        return frame, lms, offset
 
     def get_eye_state(self, frame, lms):
         if self.no_gaze:
@@ -455,27 +555,56 @@ class Tracker():
         e_x = [0,0]
         e_y = [0,0]
         scale = [0,0]
-        (right_eye, e_x[0], e_y[0], scale[0]) = self.prepare_eye(frame, lms[36:42,0:2])
+        reference = [None, None]
-        (left_eye, e_x[1], e_y[1], scale[1]) = self.prepare_eye(frame, lms[42:48,0:2])
+        angle = [0, 0]
+        face_frame, lms, offset = self.extract_face(frame, lms)
+        (right_eye, e_x[0], e_y[0], scale[0], reference[0], angle[0]) = self.prepare_eye(face_frame, frame, np.array([lms[36,0:2], lms[39,0:2]]), False)
+        (left_eye, e_x[1], e_y[1], scale[1], reference[1], angle[1]) = self.prepare_eye(face_frame, frame, np.array([lms[42,0:2], lms[45,0:2]]), True)
         if right_eye is None or left_eye is None:
             return [(1.0, 0.0, 0.0, 0.0), (1.0, 0.0, 0.0, 0.0)]
         both_eyes = np.concatenate((right_eye, left_eye))
-        results = np.array(self.gaze_model.run([], {self.input_name: both_eyes})[0])
+        results = self.gaze_model.run([], {self.input_name: both_eyes})
         open = [0, 0]
-        open[0] = results[0][3].mean()
+        open[0] = 1#results[1][0].argmax()
-        open[1] = results[1][3].mean()
+        open[1] = 1#results[1][1].argmax()
+        results = np.array(results[0])
 
         eye_state = []
         for i in range(2):
             m = int(results[i][0].argmax())
-            x = m // 8
+            x = m // 16
-            y = m % 8
+            y = m % 16
             conf = float(results[i][0][x,y])
 
-            off_y = 32.0 * logit(results[i][1][y, x], 8.0)
+            off_x = 32.0 * logit(results[i][1][x, y], 16.0)
-            off_x = 32.0 * logit(results[i][2][y, x], 8.0)
+            off_y = 32.0 * logit(results[i][2][x, y], 16.0)
-            eye_y = e_y[i] + scale[i] * (32.0 * float(x) / 8.0 + off_x)
+            if i == 1:
-            eye_x = e_x[i] + scale[i] * (32.0 * float(y) / 8.0 + off_y)
+                eye_x = 32.0 * float(x) / 16.0 + off_x
+            else:
+                eye_x = 32.0 * float(x) / 16.0 + off_x
+            eye_y = 32.0 * float(y) / 16.0 + off_y
+
+            if self.debug_gaze:
+                if i == 0:
+                    frame[int(eye_y), int(eye_x)] = (0, 0, 255)
+                    frame[int(eye_y+1), int(eye_x)] = (0, 0, 255)
+                    frame[int(eye_y+1), int(eye_x+1)] = (0, 0, 255)
+                    frame[int(eye_y), int(eye_x+1)] = (0, 0, 255)
+                else:
+                    frame[int(eye_y), 32+int(eye_x)] = (0, 0, 255)
+                    frame[int(eye_y+1), 32+int(eye_x)] = (0, 0, 255)
+                    frame[int(eye_y+1), 32+int(eye_x+1)] = (0, 0, 255)
+                    frame[int(eye_y), 32+int(eye_x+1)] = (0, 0, 255)
+
+            if i == 0:
+                eye_x = e_x[i] + scale[i][0] * eye_x
+            else:
+                eye_x = e_x[i] + scale[i][0] * (32. - eye_x)
+            eye_y = e_y[i] + scale[i][1] * eye_y
+            eye_x, eye_y = rotate(reference[i], (eye_x, eye_y), -angle[i])
+
+            eye_x = eye_x + offset[0]
+            eye_y = eye_y + offset[1]
             eye_state.append([open[i], eye_y, eye_x, conf])
 
         return eye_state