First version.
This commit is contained in:
Emiliana
commit
114ab8d44e
24 changed files with 22859 additions and 0 deletions
7
.gitignore
vendored
Normal file
7
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
*.mp4
|
||||
__pycache__/*
|
||||
Binary/*
|
||||
Samples/*
|
||||
*.zip
|
||||
*.7z
|
||||
CSharp/*
|
||||
25
LICENSE
Normal file
25
LICENSE
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
BSD 2-Clause License
|
||||
|
||||
Copyright (c) 2019, Emiliana (https://twitter.com/Emiliana_vt / https://github.com/emilianavt/)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
21
Licenses/OpenCV.txt
Normal file
21
Licenses/OpenCV.txt
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
By downloading, copying, installing or using the software you agree to this license. If you do not agree to this license, do not download, install, copy or use the software.
|
||||
|
||||
License Agreement
|
||||
For Open Source Computer Vision Library
|
||||
(3-clause BSD License)
|
||||
|
||||
Copyright (C) 2000-2019, Intel Corporation, all rights reserved.
|
||||
Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved.
|
||||
Copyright (C) 2009-2016, NVIDIA Corporation, all rights reserved.
|
||||
Copyright (C) 2010-2013, Advanced Micro Devices, Inc., all rights reserved.
|
||||
Copyright (C) 2015-2016, OpenCV Foundation, all rights reserved.
|
||||
Copyright (C) 2015-2016, Itseez Inc., all rights reserved.
|
||||
Third party copyrights are property of their respective owners.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
|
||||
* Neither the names of the copyright holders nor the names of the contributors may be used to endorse or promote products derived from this software without specific prior written permission.
|
||||
|
||||
This software is provided by the copyright holders and contributors <20>gas is<69>h and any express or implied warranties, including, but not limited to, the implied warranties of merchantability and fitness for a particular purpose are disclaimed. In no event shall copyright holders or contributors be liable for any direct, indirect, incidental, special, exemplary, or consequential damages (including, but not limited to, procurement of substitute goods or services; loss of use, data, or profits; or business interruption) however caused and on any theory of liability, whether in contract, strict liability, or tort (including negligence or otherwise) arising in any way out of the use of this software, even if advised of the possibility of such damage.
|
||||
25
Licenses/OpenSeeFace.txt
Normal file
25
Licenses/OpenSeeFace.txt
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
BSD 2-Clause License
|
||||
|
||||
Copyright (c) 2019, Emiliana (https://twitter.com/Emiliana_vt / https://github.com/emilianavt/)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
29
Licenses/escapi.txt
Normal file
29
Licenses/escapi.txt
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
The code used here contains small modifications by Emiliana.
|
||||
Originally from: https://github.com/jarikomppa/escapi/
|
||||
|
||||
License:
|
||||
|
||||
This is free and unencumbered software released into the public domain.
|
||||
|
||||
Anyone is free to copy, modify, publish, use, compile, sell, or
|
||||
distribute this software, either in source code form or as a compiled
|
||||
binary, for any purpose, commercial or non-commercial, and by any
|
||||
means.
|
||||
|
||||
In jurisdictions that recognize copyright laws, the author or authors
|
||||
of this software dedicate any and all copyright interest in the
|
||||
software to the public domain. We make this dedication for the benefit
|
||||
of the public at large and to the detriment of our heirs and
|
||||
successors. We intend this dedication to be an overt act of
|
||||
relinquishment in perpetuity of all present and future rights to this
|
||||
software under copyright law.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
For more information, please refer to <http://unlicense.org/>
|
||||
41
Licenses/haarcascade_frontalface_alt2.txt
Normal file
41
Licenses/haarcascade_frontalface_alt2.txt
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
Tree-based 20x20 gentle adaboost frontal face detector.
|
||||
Created by Rainer Lienhart.
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
|
||||
|
||||
By downloading, copying, installing or using the software you agree to this license.
|
||||
If you do not agree to this license, do not download, install,
|
||||
copy or use the software.
|
||||
|
||||
|
||||
Intel License Agreement
|
||||
For Open Source Computer Vision Library
|
||||
|
||||
Copyright (C) 2000, Intel Corporation, all rights reserved.
|
||||
Third party copyrights are property of their respective owners.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification,
|
||||
are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistribution's of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
|
||||
* Redistribution's in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
* The name of Intel Corporation may not be used to endorse or promote products
|
||||
derived from this software without specific prior written permission.
|
||||
|
||||
This software is provided by the copyright holders and contributors "as is" and
|
||||
any express or implied warranties, including, but not limited to, the implied
|
||||
warranties of merchantability and fitness for a particular purpose are disclaimed.
|
||||
In no event shall the Intel Corporation or contributors be liable for any direct,
|
||||
indirect, incidental, special, exemplary, or consequential damages
|
||||
(including, but not limited to, procurement of substitute goods or services;
|
||||
loss of use, data, or profits; or business interruption) however caused
|
||||
and on any theory of liability, whether in contract, strict liability,
|
||||
or tort (including negligence or otherwise) arising in any way out of
|
||||
the use of this software, even if advised of the possibility of such damage.
|
||||
21
Licenses/onnxruntime.txt
Normal file
21
Licenses/onnxruntime.txt
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
MIT License
|
||||
|
||||
Copyright (c) 2018 Microsoft Corporation
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
114
README.md
Normal file
114
README.md
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
# Overview
|
||||
|
||||
This project implements a facial landmark detection model based on ShuffleNetV2.
|
||||
|
||||
As Pytorch 1.3 CPU inference speed on Windows is very low, the model was converted to ONNX format. Using [onnxruntime](https://github.com/microsoft/onnxruntime) it can run at 30 - 60 fps tracking a single face. A third and smaller model runs at over 60 fps tracking a single face on a single CPU core.
|
||||
|
||||
If anyone is curious, the name is a silly pun on the open seas and seeing faces. There's no deeper meaning.
|
||||
|
||||
Thanks to [@Virtual_Deat](https://twitter.com/Virtual_Deat) for helping me test everything.
|
||||
|
||||
[Unity sample video](https://twitter.com/emiliana_vt/status/1210622149314203648) | [Sample video](https://www.youtube.com/watch?v=AOPHiAp9DBE) | [Sample video](https://www.youtube.com/watch?v=-cBSuHGdBWQ)
|
||||
|
||||
# Usage
|
||||
|
||||
The face tracking itself is done by the `facetracker.py` Python 3.7 script. It is a commandline program, so you should start it manually from cmd or write a batch file to start it.
|
||||
|
||||
The script will perform the tracking on a webcam or video file and send the tracking data over UDP. This design also allows tracking to be done on a separate PC from the one who uses the tracking information. This can be useful to enhance performance and to avoid accidentially revealing camera footage.
|
||||
|
||||
The provided `OpenSee` Unity component can receive these UDP packets and provides the received information through a public field called `trackingData`. The `OpenSeeShowPoints` component can visualize the landmark points of the first detected face. It also serves as an example. Please look at it to see how to properly make use of the `OpenSee` component. The UDP packets are received in a separate thread, so any components using the `trackingData` field of the `OpenSee` component should first copy the field and access this copy, because otherwise the information may get overwritten during processing.
|
||||
|
||||
Run the python script with `--help` to learn about the possible options you can set.
|
||||
|
||||
python facetracker.py --help
|
||||
|
||||
A simple demonstration can be achieved by creating a new scene in Unity, adding an empty game object and both the `OpenSee` and `OpenSeeShowPoints` components to it. While the scene is playing, run the face tracker on a video file:
|
||||
|
||||
python facetracker.py --visualize 1 --max-threads 2 -c video.mp4
|
||||
|
||||
This way the tracking script will output its own tracking visualization while also demonstrating the transmission of tracking data to Unity.
|
||||
|
||||
The included `OpenSeeLauncher` component allows starting the face tracker program from Unity. It is designed to work with the pyinstaller created executable distributed in the binary release bundles.
|
||||
|
||||
# General notes
|
||||
|
||||
* The tracking seems to be quite robust even with partial occlusion of the face, glasses or bad lighting conditions.
|
||||
* There is no gaze direction tracking, and the model seems prone to thinking eyes are opened, even when they are closed.
|
||||
* Depending on the frame rate, face tracking can easily use up a whole CPU core. At 30fps for a single face, it should still use less than 100% of one core on a decent CPU.
|
||||
* When setting the number of faces to track to a higher number than the number of faces actually in view, the OpenCV face detecter will attempt to find new faces every `--scan-every` frames. It can be quite slow, so try to set `--faces` no higher than the actual number of faces you are tracking.
|
||||
|
||||
# Models
|
||||
|
||||
Three pretrained models are included. Using the `--model` switch, it is possible to select them for tracking.
|
||||
|
||||
* Model **0**: This is the original model I trained. It is the most reliable for general 3D pose tracking, but it has difficulties with eyebrow and eye positions.
|
||||
* Model **1** (default): This model trades off a little bit of robustness for better facial feature tracking.
|
||||
* Model **2**: This is a smaller model that can run much faster, easily reaching 60 - 90 fps on a single CPU core. However, the tracking robustness and accuracy are lower.
|
||||
|
||||
# Release builds
|
||||
|
||||
The builds in the release section of this repository contain a `facetracker.exe` inside a `Binary` folder that was built using `pyinstaller` and contains all required dependencies.
|
||||
|
||||
To run it, at least the `models` and `escapi` folders have to be placed in the same folder as `facetracker.exe`.
|
||||
|
||||
When distributing it, you should also distribute the `Licenses` folder along with it to make sure you conform to requirements set forth by some of the third party libraries. Unused models can be removed from redistributed packages without issue.
|
||||
|
||||
# Dependencies
|
||||
|
||||
* Python 3.7
|
||||
* ONNX Runtime
|
||||
* OpenCV
|
||||
* Pillow
|
||||
* Numpy
|
||||
|
||||
The required libraries can be installed using pip:
|
||||
|
||||
pip install onnxruntime opencv-python pillow numpy
|
||||
|
||||
# References
|
||||
|
||||
## Training dataset
|
||||
|
||||
The model was trained on a 66 point version of the [LS3D-W](https://www.adrianbulat.com/face-alignment) dataset.
|
||||
|
||||
@inproceedings{bulat2017far,
|
||||
title={How far are we from solving the 2D \& 3D Face Alignment problem? (and a dataset of 230,000 3D facial landmarks)},
|
||||
author={Bulat, Adrian and Tzimiropoulos, Georgios},
|
||||
booktitle={International Conference on Computer Vision},
|
||||
year={2017}
|
||||
}
|
||||
|
||||
Additional training has been done on the WFLW dataset after reducing it to 66 points and replacing the contour points and tip of the nose with points predicted by the model trained up to this point. This additional training is done to improve fitting to eyes and eyebrows.
|
||||
|
||||
@inproceedings{wayne2018lab,
|
||||
author = {Wu, Wayne and Qian, Chen and Yang, Shuo and Wang, Quan and Cai, Yici and Zhou, Qiang},
|
||||
title = {Look at Boundary: A Boundary-Aware Face Alignment Algorithm},
|
||||
booktitle = {CVPR},
|
||||
month = June,
|
||||
year = {2018}
|
||||
}
|
||||
|
||||
The original model can be selected with `--model 0` instead.
|
||||
|
||||
## Algorithm
|
||||
|
||||
The algorithm is inspired by:
|
||||
|
||||
* [Designing Neural Network Architectures for Different Applications: From Facial Landmark Tracking to Lane Departure Warning System](https://www.synopsys.com/designware-ip/technical-bulletin/ulsee-designing-neural-network.html) by YiTa Wu, Vice President of Engineering, ULSee
|
||||
* [Real-time Human Pose Estimation in the Browser with TensorFlow.js](https://blog.tensorflow.org/2018/05/real-time-human-pose-estimation-in.html)
|
||||
* [U-Net: Convolutional Networks for Biomedical Image Segmentation](https://lmb.informatik.uni-freiburg.de/people/ronneber/u-net/) by Olaf Ronneberger, Philipp Fischer, Thomas Brox
|
||||
* [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) by Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam
|
||||
* [ShuffleNet V2: Practical Guidelines for Efficient CNN Architecture Design](https://arxiv.org/abs/1807.11164) by Ningning Ma, Xiangyu Zhang, Hai-Tao Zheng, Jian Sun
|
||||
|
||||
The ShuffleNet V2 code is taken from `torchvision`.
|
||||
|
||||
For all training after the first model, a modified version of [Adaptive Wing Loss](https://github.com/tankrant/Adaptive-Wing-Loss) was used.
|
||||
|
||||
* [Adaptive Wing Loss for Robust Face Alignment via Heatmap Regression](https://arxiv.org/abs/1904.07399) by Xinyao Wang, Liefeng Bo, Li Fuxin
|
||||
|
||||
# License
|
||||
|
||||
The code and model are distributed under the BSD 2-clause license.
|
||||
|
||||
`models/haarcascade_frontalface_alt2.xml` is distributed under its own license. You can find licenses of third party libraries used for binary builds in the `Licenses` folder.
|
||||
|
||||
170
Unity/OpenSee.cs
Normal file
170
Unity/OpenSee.cs
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Threading;
|
||||
using UnityEngine;
|
||||
|
||||
namespace OpenSee {
|
||||
|
||||
public class OpenSee : MonoBehaviour {
|
||||
[Header("UDP server settings")]
|
||||
[Tooltip("To listen for remote connections, change this to 0.0.0.0 or your actual IP on the desired interface")]
|
||||
public string listenAddress = "127.0.0.1";
|
||||
[Tooltip("This is the port the server will listen for tracking packets on")]
|
||||
public int listenPort = 11573;
|
||||
|
||||
private const int nPoints = 66;
|
||||
private const int packetFrameSize = 8 + 1 + 4 + 4 * 4 + 3 * 4 + 3 * 4 + 4 * 66 + 4 * 2 * 66 + 4 * 3 * 66;
|
||||
|
||||
[Header("Tracking data")]
|
||||
[Tooltip("This is an informational property that tells you how many packets have been received")]
|
||||
public int receivedPackets = 0;
|
||||
[Tooltip("This contains the actual tracking data")]
|
||||
public OpenSeeData[] trackingData = null;
|
||||
|
||||
[System.Serializable]
|
||||
public class OpenSeeData {
|
||||
[Tooltip("The time this tracking data was captured at.")]
|
||||
public double time;
|
||||
[Tooltip("This field tells you if 3D points have been successfully estimated from the 2D points. If this is false, do not rely on pose or 3D data.")]
|
||||
public bool got3DPoints;
|
||||
[Tooltip("This field contains the error for fitting the original 3D points. It shouldn't matter much, but it it is very high, something is probably wrong")]
|
||||
public float fit3DError;
|
||||
[Tooltip("This is the rotation vector for the 3D points to turn into the estimated face pose.")]
|
||||
public Vector3 rotation;
|
||||
[Tooltip("This is the translation vector for the 3D points to turn into the estimated face pose.")]
|
||||
public Vector3 translation;
|
||||
[Tooltip("This is the raw rotation quaternion calculated from the OpenCV rotation matrix. It does not match Unity's coordinate system, but it still might be useful.")]
|
||||
public Quaternion rawQuaternion;
|
||||
[Tooltip("This is the raw rotation euler angles calculated by OpenCV from the rotation matrix. It does not match Unity's coordinate system, but it still might be useful.")]
|
||||
public Vector3 rawEuler;
|
||||
[Tooltip("This field tells you how certain the tracker is.")]
|
||||
public float[] confidence;
|
||||
[Tooltip("These are the detected face landmarks in image coordinates.")]
|
||||
public Vector2[] points;
|
||||
[Tooltip("These are 3D points estimated from the 2D points. The should be rotation and translation compensated.")]
|
||||
public Vector3[] points3D;
|
||||
|
||||
public OpenSeeData() {
|
||||
confidence = new float[nPoints];
|
||||
points = new Vector2[nPoints];
|
||||
points3D = new Vector3[nPoints];
|
||||
}
|
||||
|
||||
private float readFloat(byte[] b, ref int o) {
|
||||
float v = System.BitConverter.ToSingle(b, o);
|
||||
o += 4;
|
||||
return v;
|
||||
}
|
||||
|
||||
private Quaternion readQuaternion(byte[] b, ref int o) {
|
||||
float x = readFloat(b, ref o);
|
||||
float y = readFloat(b, ref o);
|
||||
float z = readFloat(b, ref o);
|
||||
float w = readFloat(b, ref o);
|
||||
Quaternion q = new Quaternion(x, y, z, w);
|
||||
return q;
|
||||
}
|
||||
|
||||
private Vector3 readVector3(byte[] b, ref int o) {
|
||||
Vector3 v = new Vector3(readFloat(b, ref o), -readFloat(b, ref o), readFloat(b, ref o));
|
||||
return v;
|
||||
}
|
||||
|
||||
private Vector2 readVector2(byte[] b, ref int o) {
|
||||
Vector2 v = new Vector2(readFloat(b, ref o), readFloat(b, ref o));
|
||||
return v;
|
||||
}
|
||||
|
||||
public void readFromPacket(byte[] b, int o) {
|
||||
time = System.BitConverter.ToDouble(b, o);
|
||||
o += 8;
|
||||
|
||||
byte got3D = b[o];
|
||||
o++;
|
||||
got3DPoints = false;
|
||||
if (got3D != 0)
|
||||
got3DPoints = true;
|
||||
|
||||
fit3DError = readFloat(b, ref o);
|
||||
rawQuaternion = readQuaternion(b, ref o);
|
||||
Quaternion convertedQuaternion = new Quaternion(-rawQuaternion.x, rawQuaternion.y, -rawQuaternion.z, rawQuaternion.w);
|
||||
rawEuler = readVector3(b, ref o);
|
||||
|
||||
rotation = rawEuler;
|
||||
rotation.z = (rotation.z + 90) % 360;
|
||||
rotation.x = -(rotation.x + 180) % 360;
|
||||
|
||||
float x = readFloat(b, ref o);
|
||||
float y = readFloat(b, ref o);
|
||||
float z = readFloat(b, ref o);
|
||||
translation = new Vector3(y, -x, z);
|
||||
|
||||
for (int i = 0; i < nPoints; i++) {
|
||||
confidence[i] = readFloat(b, ref o);
|
||||
}
|
||||
|
||||
for (int i = 0; i < nPoints; i++) {
|
||||
points[i] = readVector2(b, ref o);
|
||||
}
|
||||
|
||||
for (int i = 0; i < nPoints; i++) {
|
||||
points3D[i] = readVector3(b, ref o);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Socket socket;
|
||||
private byte[] buffer;
|
||||
private Thread receiveThread = null;
|
||||
private volatile bool stopReception = false;
|
||||
|
||||
void performReception() {
|
||||
EndPoint senderRemote = new IPEndPoint(IPAddress.Any, 0);
|
||||
while (!stopReception) {
|
||||
try {
|
||||
int receivedBytes = socket.ReceiveFrom(buffer, SocketFlags.None, ref senderRemote);
|
||||
if (receivedBytes < 1 || receivedBytes % packetFrameSize != 0) {
|
||||
continue;
|
||||
}
|
||||
receivedPackets++;
|
||||
int i = 0;
|
||||
OpenSeeData[] newData = new OpenSeeData[receivedBytes / packetFrameSize];
|
||||
for (int offset = 0; offset < receivedBytes; offset += packetFrameSize) {
|
||||
newData[i] = new OpenSeeData();
|
||||
newData[i].readFromPacket(buffer, offset);
|
||||
i++;
|
||||
}
|
||||
trackingData = newData;
|
||||
} catch { }
|
||||
}
|
||||
}
|
||||
|
||||
void Start () {
|
||||
buffer = new byte[65535];
|
||||
|
||||
socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
|
||||
IPAddress ip;
|
||||
IPAddress.TryParse(listenAddress, out ip);
|
||||
socket.Bind(new IPEndPoint(ip, listenPort));
|
||||
socket.ReceiveTimeout = 15;
|
||||
|
||||
receiveThread = new Thread(() => performReception());
|
||||
receiveThread.Start();
|
||||
}
|
||||
|
||||
void Update () {
|
||||
}
|
||||
|
||||
void OnApplicationQuit() {
|
||||
if (receiveThread != null) {
|
||||
stopReception = true;
|
||||
receiveThread.Join();
|
||||
stopReception = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
150
Unity/OpenSeeJobManagement.cs
Normal file
150
Unity/OpenSeeJobManagement.cs
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
// From: https://stackoverflow.com/a/9164742
|
||||
// Is this better? https://stackoverflow.com/a/36648201
|
||||
namespace OpenSee
|
||||
{
|
||||
public class Win32 {
|
||||
[DllImport("kernel32.dll", SetLastError = true)] [return: MarshalAs(UnmanagedType.Bool)]
|
||||
static extern bool CloseHandle(IntPtr hObject);
|
||||
}
|
||||
|
||||
public class Job : IDisposable
|
||||
{
|
||||
[DllImport("kernel32.dll", CharSet = CharSet.Unicode)]
|
||||
static extern IntPtr CreateJobObject(IntPtr a, string lpName);
|
||||
|
||||
[DllImport("kernel32.dll")]
|
||||
static extern bool SetInformationJobObject(IntPtr hJob, JobObjectInfoType infoType, IntPtr lpJobObjectInfo, UInt32 cbJobObjectInfoLength);
|
||||
|
||||
[DllImport("kernel32.dll", SetLastError = true)]
|
||||
static extern bool AssignProcessToJobObject(IntPtr job, IntPtr process);
|
||||
|
||||
[DllImport("kernel32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
static extern bool CloseHandle(IntPtr hObject);
|
||||
|
||||
private IntPtr handle;
|
||||
private bool disposed;
|
||||
|
||||
public Job()
|
||||
{
|
||||
handle = CreateJobObject(IntPtr.Zero, null);
|
||||
|
||||
var info = new JOBOBJECT_BASIC_LIMIT_INFORMATION
|
||||
{
|
||||
LimitFlags = 0x2000
|
||||
};
|
||||
|
||||
var extendedInfo = new JOBOBJECT_EXTENDED_LIMIT_INFORMATION
|
||||
{
|
||||
BasicLimitInformation = info
|
||||
};
|
||||
|
||||
int length = Marshal.SizeOf(typeof(JOBOBJECT_EXTENDED_LIMIT_INFORMATION));
|
||||
IntPtr extendedInfoPtr = Marshal.AllocHGlobal(length);
|
||||
Marshal.StructureToPtr(extendedInfo, extendedInfoPtr, false);
|
||||
|
||||
if (!SetInformationJobObject(handle, JobObjectInfoType.ExtendedLimitInformation, extendedInfoPtr, (uint)length))
|
||||
throw new Exception(string.Format("Unable to set information. Error: {0}", Marshal.GetLastWin32Error()));
|
||||
|
||||
Marshal.FreeHGlobal(extendedInfoPtr);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
private void Dispose(bool disposing)
|
||||
{
|
||||
if (disposed)
|
||||
return;
|
||||
|
||||
if (disposing) { }
|
||||
|
||||
Close();
|
||||
disposed = true;
|
||||
}
|
||||
|
||||
public void Close()
|
||||
{
|
||||
CloseHandle(handle);
|
||||
handle = IntPtr.Zero;
|
||||
}
|
||||
|
||||
public bool AddProcess(IntPtr processHandle)
|
||||
{
|
||||
return AssignProcessToJobObject(handle, processHandle);
|
||||
}
|
||||
|
||||
public bool AddProcess(int processId)
|
||||
{
|
||||
return AddProcess(Process.GetProcessById(processId).Handle);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#region Helper classes
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
struct IO_COUNTERS
|
||||
{
|
||||
public UInt64 ReadOperationCount;
|
||||
public UInt64 WriteOperationCount;
|
||||
public UInt64 OtherOperationCount;
|
||||
public UInt64 ReadTransferCount;
|
||||
public UInt64 WriteTransferCount;
|
||||
public UInt64 OtherTransferCount;
|
||||
}
|
||||
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
struct JOBOBJECT_BASIC_LIMIT_INFORMATION
|
||||
{
|
||||
public Int64 PerProcessUserTimeLimit;
|
||||
public Int64 PerJobUserTimeLimit;
|
||||
public UInt32 LimitFlags;
|
||||
public UIntPtr MinimumWorkingSetSize;
|
||||
public UIntPtr MaximumWorkingSetSize;
|
||||
public UInt32 ActiveProcessLimit;
|
||||
public UIntPtr Affinity;
|
||||
public UInt32 PriorityClass;
|
||||
public UInt32 SchedulingClass;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct SECURITY_ATTRIBUTES
|
||||
{
|
||||
public UInt32 nLength;
|
||||
public IntPtr lpSecurityDescriptor;
|
||||
public Int32 bInheritHandle;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
struct JOBOBJECT_EXTENDED_LIMIT_INFORMATION
|
||||
{
|
||||
public JOBOBJECT_BASIC_LIMIT_INFORMATION BasicLimitInformation;
|
||||
public IO_COUNTERS IoInfo;
|
||||
public UIntPtr ProcessMemoryLimit;
|
||||
public UIntPtr JobMemoryLimit;
|
||||
public UIntPtr PeakProcessMemoryUsed;
|
||||
public UIntPtr PeakJobMemoryUsed;
|
||||
}
|
||||
|
||||
public enum JobObjectInfoType
|
||||
{
|
||||
AssociateCompletionPortInformation = 7,
|
||||
BasicLimitInformation = 2,
|
||||
BasicUIRestrictions = 4,
|
||||
EndOfJobTimeInformation = 6,
|
||||
ExtendedLimitInformation = 9,
|
||||
SecurityLimitInformation = 5,
|
||||
GroupInformation = 11
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
283
Unity/OpenSeeLauncher.cs
Normal file
283
Unity/OpenSeeLauncher.cs
Normal file
|
|
@ -0,0 +1,283 @@
|
|||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.RegularExpressions;
|
||||
using UnityEngine;
|
||||
|
||||
namespace OpenSee {
|
||||
|
||||
public class OpenSeeLauncher : MonoBehaviour {
|
||||
[Header("Settings")]
|
||||
[Tooltip("If this is left empty, it will default to the OpenSee component on the same game object.")]
|
||||
public OpenSee openSeeTarget;
|
||||
[Tooltip("The numeric index of the camera as in the list returned by the ListCameras function.")]
|
||||
public int cameraIndex = -1;
|
||||
[Tooltip("The path to a video file that should be used as input for the tracker instead of a camera")]
|
||||
public string videoPath = "";
|
||||
[Tooltip("Should facetracker.exe be started automatically with the scene?")]
|
||||
public bool autoStart = false;
|
||||
[Tooltip("If this is enabled, available cameras are automatically printed to the debug console.")]
|
||||
public bool autoListCameras = false;
|
||||
[Tooltip("If this is enabled, no output from the tracker will be printed to the debug console. This cannot be changed while the tracker is running.")]
|
||||
public bool dontPrint = true;
|
||||
[Tooltip("The path to \"facetracker.exe\".")]
|
||||
public string exePath = "facetracker.exe";
|
||||
[Tooltip("The path, where the .onnx model files can be found")]
|
||||
public string modelPath = "models\\";
|
||||
[Tooltip("Additional options that should be passed to the face tracker.")]
|
||||
public string[] commandlineOptions = new string[] { "--silent", "1", "--max-threads", "2", "--model", "1" };
|
||||
[Header("Runtime information")]
|
||||
[Tooltip("This field shows if the tracker is currently alive.")]
|
||||
public bool trackerAlive = false;
|
||||
|
||||
private string ip;
|
||||
private Process trackerProcess = null;
|
||||
private StringBuilder trackerSB = null;
|
||||
private bool dontPrintNow = false;
|
||||
private Job job = null;
|
||||
|
||||
private bool CheckSetup(bool requireTarget) {
|
||||
if (openSeeTarget == null)
|
||||
openSeeTarget = GetComponent<OpenSee>();
|
||||
if (openSeeTarget == null) {
|
||||
UnityEngine.Debug.LogError("No openSeeTarget is set.");
|
||||
return false;
|
||||
}
|
||||
IPAddress address;
|
||||
if (!File.Exists(exePath)) {
|
||||
UnityEngine.Debug.LogError("Facetracker executable cannot be found.");
|
||||
return false;
|
||||
}
|
||||
if (!Directory.Exists(modelPath)) {
|
||||
UnityEngine.Debug.LogError("Model directory cannot be found.");
|
||||
return false;
|
||||
}
|
||||
if (requireTarget) {
|
||||
ip = openSeeTarget.listenAddress;
|
||||
if (!IPAddress.TryParse(ip, out address)) {
|
||||
UnityEngine.Debug.LogError("No valid IP address was given in the OpenSee component.");
|
||||
return false;
|
||||
}
|
||||
if (openSeeTarget.listenPort < 1 || openSeeTarget.listenPort > 65535) {
|
||||
UnityEngine.Debug.LogError("No valid port was given in the OpenSee component.");
|
||||
return false;
|
||||
}
|
||||
if (cameraIndex >= 0 && videoPath != "") {
|
||||
UnityEngine.Debug.LogError("Entering both a camera index and a video filename is not valid.");
|
||||
return false;
|
||||
}
|
||||
if (cameraIndex < 0 && !File.Exists(videoPath)) {
|
||||
if (videoPath != "") {
|
||||
UnityEngine.Debug.LogError("The given video file does not exist.");
|
||||
} else {
|
||||
UnityEngine.Debug.LogError("No camera or video file was given.");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (job == null) {
|
||||
job = new Job();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public string[] ListCameras() {
|
||||
if (!CheckSetup(false))
|
||||
return null;
|
||||
|
||||
StringBuilder stringBuilder;
|
||||
ProcessStartInfo processStartInfo;
|
||||
Process process;
|
||||
|
||||
stringBuilder = new StringBuilder();
|
||||
|
||||
processStartInfo = new ProcessStartInfo();
|
||||
processStartInfo.CreateNoWindow = true;
|
||||
processStartInfo.RedirectStandardOutput = true;
|
||||
processStartInfo.RedirectStandardInput = true;
|
||||
processStartInfo.RedirectStandardError = true;
|
||||
processStartInfo.UseShellExecute = false;
|
||||
processStartInfo.FileName = exePath;
|
||||
processStartInfo.Arguments = "--list-cameras 2";
|
||||
|
||||
process = new Process();
|
||||
process.StartInfo = processStartInfo;
|
||||
process.EnableRaisingEvents = true;
|
||||
process.OutputDataReceived += new DataReceivedEventHandler(delegate(object sender, DataReceivedEventArgs e) {
|
||||
stringBuilder.Append(e.Data);
|
||||
stringBuilder.Append("\n");
|
||||
});
|
||||
process.Start();
|
||||
job.AddProcess(process.Handle);
|
||||
process.BeginOutputReadLine();
|
||||
process.WaitForExit();
|
||||
process.CancelOutputRead();
|
||||
|
||||
string cameraInfo = stringBuilder.ToString();
|
||||
string[] cameras = cameraInfo.Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.RemoveEmptyEntries);
|
||||
return cameras;
|
||||
}
|
||||
|
||||
// From: http://csharptest.net/529/how-to-correctly-escape-command-line-arguments-in-c/index.html
|
||||
private static string EscapeArguments(params string[] args)
|
||||
{
|
||||
StringBuilder arguments = new StringBuilder();
|
||||
Regex invalidChar = new Regex("[\x00\x0a\x0d]");// these can not be escaped
|
||||
Regex needsQuotes = new Regex(@"\s|""");// contains whitespace or two quote characters
|
||||
Regex escapeQuote = new Regex(@"(\\*)(""|$)");// one or more '\' followed with a quote or end of string
|
||||
for (int carg = 0; args != null && carg < args.Length; carg++)
|
||||
{
|
||||
if (args[carg] == null) { throw new ArgumentNullException("args[" + carg + "]"); }
|
||||
if (invalidChar.IsMatch(args[carg])) { throw new ArgumentOutOfRangeException("args[" + carg + "]"); }
|
||||
if (args[carg] == String.Empty) { arguments.Append("\"\""); }
|
||||
else if (!needsQuotes.IsMatch(args[carg])) { arguments.Append(args[carg]); }
|
||||
else
|
||||
{
|
||||
arguments.Append('"');
|
||||
arguments.Append(escapeQuote.Replace(args[carg], m => m.Groups[1].Value + m.Groups[1].Value + (m.Groups[2].Value == "\"" ? "\\\"" : "")));
|
||||
arguments.Append('"');
|
||||
}
|
||||
if (carg + 1 < args.Length)
|
||||
arguments.Append(' ');
|
||||
}
|
||||
return arguments.ToString();
|
||||
}
|
||||
|
||||
public bool StartTracker() {
|
||||
if (!CheckSetup(true))
|
||||
return false;
|
||||
|
||||
ProcessStartInfo processStartInfo;
|
||||
|
||||
processStartInfo = new ProcessStartInfo();
|
||||
processStartInfo.CreateNoWindow = true;
|
||||
processStartInfo.RedirectStandardOutput = true;
|
||||
processStartInfo.RedirectStandardInput = true;
|
||||
processStartInfo.RedirectStandardError = true;
|
||||
processStartInfo.UseShellExecute = false;
|
||||
processStartInfo.FileName = exePath;
|
||||
|
||||
List<string> arguments = new List<string>();
|
||||
arguments.Add("--ip");
|
||||
arguments.Add(ip);
|
||||
arguments.Add("--port");
|
||||
arguments.Add(openSeeTarget.listenPort.ToString());
|
||||
arguments.Add("--model-dir");
|
||||
arguments.Add(modelPath);
|
||||
arguments.Add("--capture");
|
||||
if (videoPath != "" && File.Exists(videoPath))
|
||||
arguments.Add(videoPath);
|
||||
else
|
||||
arguments.Add(cameraIndex.ToString());
|
||||
foreach (string argument in commandlineOptions) {
|
||||
arguments.Add(argument);
|
||||
}
|
||||
string argumentString = EscapeArguments(arguments.ToArray());
|
||||
processStartInfo.Arguments = argumentString;
|
||||
|
||||
if (trackerProcess != null && !trackerProcess.HasExited) {
|
||||
trackerProcess.CloseMainWindow();
|
||||
trackerProcess.Close();
|
||||
if (!dontPrintNow) {
|
||||
try {
|
||||
trackerProcess.CancelOutputRead();
|
||||
trackerProcess.CancelErrorRead();
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
trackerSB = new StringBuilder();
|
||||
trackerProcess = new Process();
|
||||
trackerProcess.StartInfo = processStartInfo;
|
||||
dontPrintNow = dontPrint;
|
||||
if (!dontPrintNow) {
|
||||
trackerProcess.EnableRaisingEvents = true;
|
||||
trackerProcess.OutputDataReceived += new DataReceivedEventHandler(delegate(object sender, DataReceivedEventArgs e) {
|
||||
trackerSB.Append(e.Data);
|
||||
trackerSB.Append("\n");
|
||||
});
|
||||
trackerProcess.ErrorDataReceived += new DataReceivedEventHandler(delegate(object sender, DataReceivedEventArgs e) {
|
||||
trackerSB.Append(e.Data);
|
||||
trackerSB.Append("\n");
|
||||
});
|
||||
}
|
||||
trackerProcess.Start();
|
||||
job.AddProcess(trackerProcess.Handle);
|
||||
if (!dontPrintNow) {
|
||||
trackerProcess.BeginOutputReadLine();
|
||||
trackerProcess.BeginErrorReadLine();
|
||||
}
|
||||
|
||||
return !trackerProcess.HasExited;
|
||||
}
|
||||
|
||||
public void StopTracker() {
|
||||
if (trackerProcess != null && !trackerProcess.HasExited) {
|
||||
trackerProcess.CloseMainWindow();
|
||||
trackerProcess.Close();
|
||||
if (!dontPrintNow) {
|
||||
try {
|
||||
trackerProcess.CancelOutputRead();
|
||||
trackerProcess.CancelErrorRead();
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
trackerProcess = null;
|
||||
}
|
||||
|
||||
public void Start() {
|
||||
if (autoStart)
|
||||
StartTracker();
|
||||
if (autoListCameras) {
|
||||
string[] cameras = ListCameras();
|
||||
if (cameras == null || cameras.Length < 1) {
|
||||
UnityEngine.Debug.Log("No cameras found.");
|
||||
return;
|
||||
}
|
||||
UnityEngine.Debug.Log("Cameras:");
|
||||
for (int i = 0; i < cameras.Length; i++) {
|
||||
UnityEngine.Debug.Log(i + " = " + cameras[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Update() {
|
||||
if (trackerProcess != null)
|
||||
trackerAlive = !trackerProcess.HasExited;
|
||||
else {
|
||||
trackerAlive = false;
|
||||
return;
|
||||
}
|
||||
if (dontPrintNow || trackerSB == null)
|
||||
return;
|
||||
int len = trackerSB.Length;
|
||||
if (len > 0) {
|
||||
string output = trackerSB.ToString();
|
||||
UnityEngine.Debug.Log("Tracker: " + output);
|
||||
trackerSB.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
private void CleanJob() {
|
||||
if (job != null) {
|
||||
job.Dispose();
|
||||
job = null;
|
||||
}
|
||||
}
|
||||
|
||||
public void OnDestroy() {
|
||||
StopTracker();
|
||||
CleanJob();
|
||||
}
|
||||
|
||||
public void OnApplicationQuit() {
|
||||
StopTracker();
|
||||
CleanJob();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
109
Unity/OpenSeeShowPoints.cs
Normal file
109
Unity/OpenSeeShowPoints.cs
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using UnityEngine;
|
||||
|
||||
namespace OpenSee {
|
||||
|
||||
public class OpenSeeShowPoints : MonoBehaviour {
|
||||
public OpenSee openSee = null;
|
||||
public bool show3DPoints = true;
|
||||
public bool applyTranslation = false;
|
||||
public bool applyRotation = false;
|
||||
private OpenSee.OpenSeeData[] openSeeData;
|
||||
private GameObject[] gameObjects;
|
||||
private GameObject centerBall;
|
||||
private double updated = 0.0;
|
||||
|
||||
void Start () {
|
||||
if (openSee == null) {
|
||||
openSee = GetComponent<OpenSee>();
|
||||
}
|
||||
gameObjects = new GameObject[66];
|
||||
for (int i = 0; i < 66; i++) {
|
||||
gameObjects[i] = GameObject.CreatePrimitive(PrimitiveType.Sphere);
|
||||
gameObjects[i].name = "Point " + (i + 1);
|
||||
gameObjects[i].transform.SetParent(transform);
|
||||
gameObjects[i].transform.localScale = new Vector3(0.025f, 0.025f, 0.025f);
|
||||
}
|
||||
centerBall = GameObject.CreatePrimitive(PrimitiveType.Sphere);
|
||||
centerBall.name = "Center";
|
||||
centerBall.transform.SetParent(transform);
|
||||
centerBall.transform.localScale = new Vector3(0.1f, 0.1f, 0.1f);
|
||||
}
|
||||
|
||||
void Update () {
|
||||
if (!openSee)
|
||||
return;
|
||||
openSeeData = openSee.trackingData;
|
||||
if (openSeeData == null || openSeeData.Length < 1)
|
||||
return;
|
||||
if (openSeeData[0].time > updated) {
|
||||
updated = openSeeData[0].time;
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
if (show3DPoints) {
|
||||
centerBall.gameObject.SetActive(false);
|
||||
for (int i = 0; i < 66; i++) {
|
||||
if (openSeeData[0].got3DPoints) {
|
||||
Renderer renderer = gameObjects[i].GetComponent<Renderer>();
|
||||
renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData[0].confidence[i]));
|
||||
gameObjects[i].transform.localPosition = openSeeData[0].points3D[i];
|
||||
} else {
|
||||
Renderer renderer = gameObjects[i].GetComponent<Renderer>();
|
||||
renderer.material.SetColor("_Color", Color.cyan);
|
||||
}
|
||||
}
|
||||
if (applyTranslation) {
|
||||
transform.position = openSeeData[0].translation;
|
||||
}
|
||||
if (applyRotation) {
|
||||
transform.eulerAngles = openSeeData[0].rotation;
|
||||
}
|
||||
} else {
|
||||
centerBall.gameObject.SetActive(false);
|
||||
Vector3 center = new Vector3(0.0f, 0.0f, 0.0f);
|
||||
float minX = 10000.0f;
|
||||
float minY = 10000.0f;
|
||||
float maxX = -1.0f;
|
||||
float maxY = -1.0f;
|
||||
for (int i = 0; i < 66; i++) {
|
||||
float x = openSeeData[0].points[i].x;
|
||||
float y = -openSeeData[0].points[i].y;
|
||||
if (minX > x) {
|
||||
minX = x;
|
||||
}
|
||||
if (minY > y) {
|
||||
minY = y;
|
||||
}
|
||||
if (maxX < x) {
|
||||
maxX = x;
|
||||
}
|
||||
if (maxY < y) {
|
||||
maxY = y;
|
||||
}
|
||||
center += new Vector3(x, y, 0.0f);
|
||||
}
|
||||
center = center / 66;
|
||||
center = center - new Vector3(minX, minY, 0.0f);
|
||||
center.x = center.x / (maxX - minX);
|
||||
center.y = center.y / (maxX - minX);
|
||||
center.z = 0.5f;
|
||||
centerBall.transform.localPosition = center;
|
||||
|
||||
for (int i = 0; i < 66; i++) {
|
||||
Renderer renderer = gameObjects[i].GetComponent<Renderer>();
|
||||
renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData[0].confidence[i]));
|
||||
float x = openSeeData[0].points[i].x;
|
||||
float y = -openSeeData[0].points[i].y;
|
||||
Vector3 position = new Vector3(x, y, 0.0f);
|
||||
position = position - new Vector3(minX, minY, 0.0f);
|
||||
position.x = position.x / (maxX - minX);
|
||||
position.y = position.y / (maxX - minX);
|
||||
gameObjects[i].transform.localPosition = position;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
125
escapi.py
Normal file
125
escapi.py
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
"""
|
||||
A simple python wrapper around escapi
|
||||
|
||||
Usage:
|
||||
|
||||
from camera import Device
|
||||
|
||||
device = Deveice.connect(0, 500, 500)
|
||||
image = device.get_image()
|
||||
"""
|
||||
|
||||
import os
|
||||
import platform
|
||||
import sys
|
||||
from ctypes import *
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
import cv2
|
||||
|
||||
def resolve(name):
|
||||
f = os.path.join(os.path.dirname(__file__), name)
|
||||
return f
|
||||
|
||||
|
||||
class CAPTURE_PROPETIES:
|
||||
CAPTURE_BRIGHTNESS = 1,
|
||||
CAPTURE_CONTRAST = 2,
|
||||
CAPTURE_HUE = 3,
|
||||
CAPTURE_SATURATION = 4,
|
||||
CAPTURE_SHARPNESS = 5,
|
||||
CAPTURE_GAMMA = 6,
|
||||
CAPTURE_COLORENABLE = 7,
|
||||
CAPTURE_WHITEBALANCE = 8,
|
||||
CAPTURE_BACKLIGHTCOMPENSATION = 9,
|
||||
CAPTURE_GAIN = 10,
|
||||
CAPTURE_PAN = 11,
|
||||
CAPTURE_TILT = 12,
|
||||
CAPTURE_ROLL = 13,
|
||||
CAPTURE_ZOOM = 14,
|
||||
CAPTURE_EXPOSURE = 15,
|
||||
CAPTURE_IRIS = 16,
|
||||
CAPTURE_FOCUS = 17,
|
||||
CAPTURE_PROP_MAX = 18,
|
||||
|
||||
|
||||
class SimpleCapParms(Structure):
|
||||
_fields_ = [
|
||||
("buffer", POINTER(c_int)),
|
||||
("width", c_int),
|
||||
("height", c_int),
|
||||
("fps", c_int),
|
||||
]
|
||||
|
||||
|
||||
lib = None
|
||||
|
||||
|
||||
def init():
|
||||
global lib
|
||||
if platform.architecture()[0] == '32bit':
|
||||
dll_path = resolve(os.path.join("escapi", "escapi_x86.dll"))
|
||||
lib = cdll.LoadLibrary(dll_path)
|
||||
else:
|
||||
dll_path = resolve(os.path.join("escapi", "escapi_x64.dll"))
|
||||
lib = cdll.LoadLibrary(dll_path)
|
||||
if lib is None or lib.ESCAPIVersion() != 0xfff001:
|
||||
print("Invalid ESCAPI DLL found.")
|
||||
sys.exit(1)
|
||||
lib.initCapture.argtypes = [c_int, POINTER(SimpleCapParms)]
|
||||
lib.initCapture.restype = c_int
|
||||
lib.initCOM()
|
||||
|
||||
def count_capture_devices():
|
||||
return lib.countCaptureDevices()
|
||||
|
||||
def device_name(device):
|
||||
"""
|
||||
Get the device name for the given device
|
||||
:param device: The number of the device
|
||||
:return: The name of the device
|
||||
"""
|
||||
namearry = (c_char * 256)()
|
||||
lib.getCaptureDeviceName(device, namearry, 256)
|
||||
camearaname = namearry.value
|
||||
return camearaname
|
||||
|
||||
def init_camera(device, width, height, fps):
|
||||
global devices
|
||||
array = (width * height * c_int)()
|
||||
options = SimpleCapParms()
|
||||
options.width = width
|
||||
options.height = height
|
||||
options.fps = fps
|
||||
options.buffer = array
|
||||
lib.initCapture(device, byref(options))
|
||||
return array
|
||||
|
||||
def do_capture(device):
|
||||
lib.doCapture(device)
|
||||
|
||||
def is_capture_done(device):
|
||||
return lib.isCaptureDone(device)
|
||||
|
||||
def read(device, width, height, array):
|
||||
if is_capture_done(device):
|
||||
img = Image.frombuffer('RGBA', (width, height), array, 'raw', 'BGRA', 0, 1)
|
||||
img = np.array(img)
|
||||
img = cv2.cvtColor(img, cv2.COLOR_RGBA2BGR);
|
||||
return img
|
||||
else:
|
||||
return None
|
||||
|
||||
def get_image(device, width, height, array):
|
||||
lib.doCapture(device)
|
||||
|
||||
while lib.isCaptureDone(device) == 0:
|
||||
pass
|
||||
|
||||
img = Image.frombuffer('RGBA', (width, height), array, 'raw', 'BGRA', 0, 0)
|
||||
img = np.array(img)
|
||||
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR);
|
||||
return img
|
||||
|
||||
def deinit_camera(device):
|
||||
lib.deinitCapture(device)
|
||||
BIN
escapi/escapi_x64.dll
Normal file
BIN
escapi/escapi_x64.dll
Normal file
Binary file not shown.
BIN
escapi/escapi_x86.dll
Normal file
BIN
escapi/escapi_x86.dll
Normal file
Binary file not shown.
204
facetracker.py
Normal file
204
facetracker.py
Normal file
|
|
@ -0,0 +1,204 @@
|
|||
import os
|
||||
import sys
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument("-i", "--ip", help="Set IP address for sending tracking data", default="127.0.0.1")
|
||||
parser.add_argument("-p", "--port", type=int, help="Set port for sending tracking data", default=11573)
|
||||
if os.name == 'nt':
|
||||
parser.add_argument("-l", "--list-cameras", type=int, help="Set this to 1 to list the available cameras and quit, set this to a number higher than 1 to output only the names", default=0)
|
||||
parser.add_argument("-W", "--width", type=int, help="Set camera and raw RGB width", default=640)
|
||||
parser.add_argument("-H", "--height", type=int, help="Set camera and raw RGB height", default=360)
|
||||
parser.add_argument("-F", "--fps", type=int, help="Set camera frames per second", default=24)
|
||||
else:
|
||||
parser.add_argument("-W", "--width", type=int, help="Set raw RGB width", default=640)
|
||||
parser.add_argument("-H", "--height", type=int, help="Set raw RGB height", default=360)
|
||||
parser.add_argument("-c", "--capture", help="Set camera ID (0, 1...) or video file", default=0)
|
||||
parser.add_argument("-m", "--max-threads", type=int, help="Set the maximum number of threads", default=1)
|
||||
parser.add_argument("-t", "--threshold", type=float, help="Set minimum confidence threshold for face detection", default=0.4)
|
||||
parser.add_argument("-v", "--visualize", type=int, help="Set this to 1 to visualize the tracking or to 2 to add numbers to the point display", default=0)
|
||||
parser.add_argument("-P", "--pnp-points", type=int, help="Set this to 1 to add the 3D fitting points to the visualization", default=0)
|
||||
parser.add_argument("-s", "--silent", type=int, help="Set this to 1 to prevent text output on the console", default=0)
|
||||
parser.add_argument("--faces", type=int, help="Set the maximum number of faces (slow)", default=1)
|
||||
parser.add_argument("--scan-every", type=int, help="Set after how many frames a scan for new faces should run", default=3)
|
||||
parser.add_argument("--discard-after", type=int, help="Set the how long the tracker should keep looking for lost faces", default=10)
|
||||
parser.add_argument("--video-out", help="Set this to the filename of an AVI file to save the tracking visualization as a video", default=None)
|
||||
parser.add_argument("--raw-rgb", type=int, help="When this is set, raw RGB frames of the size given with \"-W\" and \"-H\" are read from standard input instead of reading a video", default=0)
|
||||
parser.add_argument("--log-data", help="You can set a filename to which tracking data will be logged here", default="")
|
||||
parser.add_argument("--model", type=int, help="This can be used to select between the two models with 0 being the original model that might be overall more accurate, 1 being a model that might be able to fit eye and eyebrow shapes more accurately and 2 being a much faster, but lower accuracy model", default=1, choices=[0, 1, 2])
|
||||
parser.add_argument("--model-dir", help="This can be used to specify the path to the directory containing the .onnx model files", default=None)
|
||||
parser.add_argument("--high-quality-3d", type=int, help="When set to 1, more nose points are used when estimating the face pose", default=1)
|
||||
args = parser.parse_args()
|
||||
|
||||
os.environ["OMP_NUM_THREADS"] = "1"
|
||||
|
||||
if os.name == 'nt' and args.list_cameras > 0:
|
||||
import escapi
|
||||
escapi.init()
|
||||
camera_count = escapi.count_capture_devices()
|
||||
if args.list_cameras == 1:
|
||||
print("Available cameras:")
|
||||
for i in range(camera_count):
|
||||
camera_name = escapi.device_name(i).decode()
|
||||
if args.list_cameras == 1:
|
||||
print(f"{i}: {camera_name}")
|
||||
else:
|
||||
print(camera_name)
|
||||
sys.exit(0)
|
||||
|
||||
import numpy as np
|
||||
import time
|
||||
import cv2
|
||||
import socket
|
||||
import struct
|
||||
from input_reader import InputReader, list_cameras
|
||||
from tracker import Tracker
|
||||
|
||||
target_ip = args.ip
|
||||
target_port = args.port
|
||||
|
||||
if args.faces >= 40:
|
||||
print("Transmission of tracking data over network is not supported with 40 or more faces.")
|
||||
|
||||
fps = 0
|
||||
if os.name == 'nt':
|
||||
fps = args.fps
|
||||
input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps)
|
||||
|
||||
log = None
|
||||
out = None
|
||||
first = True
|
||||
height = 0
|
||||
width = 0
|
||||
tracker = None
|
||||
sock = None
|
||||
tracking_time = 0.0
|
||||
tracking_frames = 0
|
||||
frame_count = 0
|
||||
|
||||
if args.log_data != "":
|
||||
log = open(args.log_data, "w")
|
||||
log.write("Frame,Time,Face,AverageConfidence,Success3D,PnPError,RotationQuat.X,RotationQuat.Y,RotationQuat.Z,RotationQuat.W,RVec.X,RVec.Y,RVec.Z,TVec.X,TVec.Y,TVec.Z")
|
||||
for i in range(66):
|
||||
log.write(f",Landmark[{i}].X,Landmark[{i}].Y,Landmark[{i}].Confidence")
|
||||
for i in range(66):
|
||||
log.write(f",Point3D[{i}].X,Point3D[{i}].Y,Point3D[{i}].Z")
|
||||
log.write("\r\n")
|
||||
log.flush()
|
||||
|
||||
try:
|
||||
while input_reader.is_open():
|
||||
if not input_reader.is_ready():
|
||||
time.sleep(0.001)
|
||||
continue
|
||||
|
||||
ret, frame = input_reader.read()
|
||||
if not ret:
|
||||
break
|
||||
|
||||
frame_count += 1
|
||||
now = time.time()
|
||||
|
||||
if first:
|
||||
first = False
|
||||
height, width, channels = frame.shape
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
tracker = Tracker(width, height, threshold=args.threshold, max_threads=args.max_threads, max_faces=args.faces, discard_after=args.discard_after, scan_every=args.scan_every, silent=False if args.silent == 0 else True, model_type=args.model, pnp_quality=args.high_quality_3d, model_dir=args.model_dir)
|
||||
if not args.video_out is None:
|
||||
out = cv2.VideoWriter(args.video_out, cv2.VideoWriter_fourcc('M','J','P','G'), 24, (width,height))
|
||||
|
||||
inference_start = time.time()
|
||||
faces = tracker.predict(frame)
|
||||
if len(faces) > 0:
|
||||
tracking_time += (time.time() - inference_start) / len(faces)
|
||||
tracking_frames += 1
|
||||
packet = bytearray()
|
||||
detected = False
|
||||
for face_num, (conf, lms, success_3d, pnp_error, quaternion, euler, rotation, translation, pts_3d) in enumerate(faces):
|
||||
if args.silent == 0:
|
||||
print(f"Confidence: {conf:.4f} / 3D fitting error: {pnp_error:.4f}")
|
||||
detected = True
|
||||
if not success_3d:
|
||||
pts_3d = np.zeros((66, 3), np.float32)
|
||||
packet.extend(bytearray(struct.pack("d", now)))
|
||||
packet.extend(bytearray(struct.pack("B", 1 if success_3d else 0)))
|
||||
packet.extend(bytearray(struct.pack("f", pnp_error)))
|
||||
packet.extend(bytearray(struct.pack("f", quaternion[0])))
|
||||
packet.extend(bytearray(struct.pack("f", quaternion[1])))
|
||||
packet.extend(bytearray(struct.pack("f", quaternion[2])))
|
||||
packet.extend(bytearray(struct.pack("f", quaternion[3])))
|
||||
packet.extend(bytearray(struct.pack("f", euler[0])))
|
||||
packet.extend(bytearray(struct.pack("f", euler[1])))
|
||||
packet.extend(bytearray(struct.pack("f", euler[2])))
|
||||
packet.extend(bytearray(struct.pack("f", translation[0][0])))
|
||||
packet.extend(bytearray(struct.pack("f", translation[0][1])))
|
||||
packet.extend(bytearray(struct.pack("f", translation[0][2])))
|
||||
if not log is None:
|
||||
log.write(f"{frame_count},{now},{face_num},{conf},{success_3d},{pnp_error},{quaternion[0]},{quaternion[1]},{quaternion[2]},{quaternion[3]},{rotation[0][0]},{rotation[0][1]},{rotation[0][2]},{translation[0][0]},{translation[0][1]},{translation[0][2]}")
|
||||
for (x,y,c) in lms:
|
||||
packet.extend(bytearray(struct.pack("f", c)))
|
||||
for pt_num, (x,y,c) in enumerate(lms):
|
||||
packet.extend(bytearray(struct.pack("f", y)))
|
||||
packet.extend(bytearray(struct.pack("f", x)))
|
||||
if not log is None:
|
||||
log.write(f",{y},{x},{c}")
|
||||
if args.visualize != 0 or not out is None:
|
||||
if args.visualize > 1:
|
||||
frame = cv2.putText(frame, str(pt_num), (int(y), int(x)), cv2.FONT_HERSHEY_SIMPLEX, 0.25, (255,255,0))
|
||||
if not (x < 0 or y < 0 or x >= height or y >= width):
|
||||
frame[int(x), int(y)] = (0, 0, 255)
|
||||
x += 1
|
||||
if not (x < 0 or y < 0 or x >= height or y >= width):
|
||||
frame[int(x), int(y)] = (0, 0, 255)
|
||||
y += 1
|
||||
if not (x < 0 or y < 0 or x >= height or y >= width):
|
||||
frame[int(x), int(y)] = (0, 0, 255)
|
||||
x -= 1
|
||||
if not (x < 0 or y < 0 or x >= height or y >= width):
|
||||
frame[int(x), int(y)] = (0, 0, 255)
|
||||
if args.pnp_points != 0 and (args.visualize != 0 or not out is None):
|
||||
projected = cv2.projectPoints(tracker.contour, rotation, translation, tracker.camera, tracker.dist_coeffs)
|
||||
for [(x,y)] in projected[0]:
|
||||
if not (x < 0 or y < 0 or x >= height or y >= width):
|
||||
frame[int(x), int(y)] = (0, 255, 255)
|
||||
x += 1
|
||||
if not (x < 0 or y < 0 or x >= height or y >= width):
|
||||
frame[int(x), int(y)] = (0, 255, 255)
|
||||
y += 1
|
||||
if not (x < 0 or y < 0 or x >= height or y >= width):
|
||||
frame[int(x), int(y)] = (0, 255, 255)
|
||||
x -= 1
|
||||
if not (x < 0 or y < 0 or x >= height or y >= width):
|
||||
frame[int(x), int(y)] = (0, 255, 255)
|
||||
for (x,y,z) in pts_3d:
|
||||
packet.extend(bytearray(struct.pack("f", x)))
|
||||
packet.extend(bytearray(struct.pack("f", -y)))
|
||||
packet.extend(bytearray(struct.pack("f", -z)))
|
||||
if not log is None:
|
||||
log.write(f",{x},{-y},{-z}")
|
||||
if not log is None:
|
||||
log.write("\r\n")
|
||||
log.flush()
|
||||
|
||||
if detected and len(faces) < 40:
|
||||
sock.sendto(packet, (target_ip, target_port))
|
||||
|
||||
if not out is None:
|
||||
out.write(frame)
|
||||
|
||||
if args.visualize != 0:
|
||||
cv2.imshow('Frame', frame)
|
||||
if cv2.waitKey(1) & 0xFF == ord('q'):
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
if args.silent == 0:
|
||||
print("Quitting")
|
||||
|
||||
input_reader.close()
|
||||
if not out is None:
|
||||
out.release()
|
||||
cv2.destroyAllWindows()
|
||||
|
||||
if args.silent == 0 and tracking_frames > 0:
|
||||
tracking_time = 1000 * tracking_time / tracking_frames
|
||||
print(f"Average tracking time per detected face: {tracking_time:.2f} ms")
|
||||
113
input_reader.py
Normal file
113
input_reader.py
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
import re
|
||||
import sys
|
||||
import os
|
||||
import cv2
|
||||
import numpy
|
||||
import escapi
|
||||
|
||||
def list_cameras():
|
||||
print("Available cameras:")
|
||||
|
||||
class VideoReader():
|
||||
def __init__(self, capture):
|
||||
self.cap = cv2.VideoCapture(capture)
|
||||
if self.cap is None:
|
||||
print("The video source cannot be opened")
|
||||
sys.exit(0)
|
||||
def is_open(self):
|
||||
return self.cap.isOpened()
|
||||
def is_ready(self):
|
||||
return True
|
||||
def read(self):
|
||||
return self.cap.read()
|
||||
def close(self):
|
||||
self.cap.release()
|
||||
|
||||
class CameraReader(VideoReader):
|
||||
def __init__(self, capture, width, height, fps):
|
||||
self.device = None
|
||||
self.width = width
|
||||
self.height = height
|
||||
self.fps = fps
|
||||
if os.name == 'nt':
|
||||
escapi.init()
|
||||
self.device = capture
|
||||
self.buffer = escapi.init_camera(self.device, self.width, self.height, self.fps)
|
||||
escapi.do_capture(self.device)
|
||||
else:
|
||||
super(CameraReader, self).__init__(capture)
|
||||
self.cap.set(3, width)
|
||||
self.cap.set(4, height)
|
||||
def is_open(self):
|
||||
if self.device is None:
|
||||
return super(CameraReader, self).is_open()
|
||||
return True
|
||||
def is_ready(self):
|
||||
if self.device is None:
|
||||
return super(CameraReader, self).is_ready()
|
||||
return escapi.is_capture_done(self.device)
|
||||
def read(self):
|
||||
if self.device is None:
|
||||
return super(CameraReader, self).read()
|
||||
if escapi.is_capture_done(self.device):
|
||||
image = escapi.read(self.device, self.width, self.height, self.buffer)
|
||||
escapi.do_capture(self.device)
|
||||
return True, image
|
||||
else:
|
||||
return False, None
|
||||
def close(self):
|
||||
if self.device is None:
|
||||
super(CameraReader, self).close()
|
||||
else:
|
||||
escapi.deinit_camera(self.device)
|
||||
|
||||
class RawReader:
|
||||
def __init__(self, width, height):
|
||||
self.width = int(width)
|
||||
self.height = int(height)
|
||||
|
||||
if self.width < 1 or self.height < 1:
|
||||
print("No acceptable size was given for reading raw RGB frames.")
|
||||
sys.exit(0)
|
||||
|
||||
self.len = self.width * self.height * 3
|
||||
self.open = True
|
||||
def is_open(self):
|
||||
return self.open
|
||||
def is_ready(self):
|
||||
return True
|
||||
def read(self):
|
||||
frame = bytearray()
|
||||
read_bytes = 0
|
||||
while read_bytes < self.len:
|
||||
bytes = sys.stdin.buffer.read(self.len)
|
||||
read_bytes += len(bytes)
|
||||
frame.extend(bytes)
|
||||
return np.frombuffer(frame, dtype=np.uint8).reshape((self.height, self.width, 3))
|
||||
def close(self):
|
||||
self.open = False
|
||||
|
||||
class InputReader():
|
||||
def __init__(self, capture, raw_rgb, width, height, fps):
|
||||
self.reader = None
|
||||
try:
|
||||
if os.path.exists(capture):
|
||||
self.reader = VideoReader(capture)
|
||||
elif capture == str(int(capture)):
|
||||
self.reader = CameraReader(int(capture), width, height, fps)
|
||||
else:
|
||||
self.reader = RawReader(width, height)
|
||||
except Exception as e:
|
||||
print("Error: " + str(e))
|
||||
|
||||
if self.reader is None or not self.reader.is_open():
|
||||
print("There was no valid input.")
|
||||
sys.exit(0)
|
||||
def is_open(self):
|
||||
return self.reader.is_open()
|
||||
def is_ready(self):
|
||||
return self.reader.is_ready()
|
||||
def read(self):
|
||||
return self.reader.read()
|
||||
def close(self):
|
||||
self.reader.close()
|
||||
9
make_exe.bat
Normal file
9
make_exe.bat
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
REM pyinstaller --onefile ^
|
||||
REM --add-binary escapi/escapi_x86.dll;. ^
|
||||
REM --add-binary escapi/escapi_x64.dll;. ^
|
||||
REM facetracker.py
|
||||
|
||||
pyinstaller --onedir ^
|
||||
--add-binary escapi/escapi_x86.dll;. ^
|
||||
--add-binary escapi/escapi_x64.dll;. ^
|
||||
facetracker.py
|
||||
220
model.py
Normal file
220
model.py
Normal file
|
|
@ -0,0 +1,220 @@
|
|||
# This file is not used by the tracking application
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
class DSConv2d(nn.Module):
|
||||
def __init__(self, in_planes, out_planes, kernels_per_layer=4, groups=1):
|
||||
super(DSConv2d, self).__init__()
|
||||
self.conv = nn.Sequential(
|
||||
nn.Conv2d(in_planes, in_planes * kernels_per_layer, kernel_size=3, padding=1, groups=in_planes),
|
||||
nn.Conv2d(in_planes * kernels_per_layer, out_planes, kernel_size=1, groups=groups)
|
||||
)
|
||||
def forward(self, x):
|
||||
x = self.conv(x)
|
||||
return x
|
||||
|
||||
class UNetUp(nn.Module):
|
||||
def __init__(self, in_channels, residual_in_channels, out_channels, size):
|
||||
super(UNetUp, self).__init__()
|
||||
self.up = nn.Upsample(size=size, mode='bilinear', align_corners=True)
|
||||
self.conv = DSConv2d(in_channels + residual_in_channels, out_channels, 1, 1)
|
||||
def forward(self, x1, x2):
|
||||
x1 = self.up(x1)
|
||||
x = torch.cat([x2, x1], dim=1)
|
||||
x = self.conv(x)
|
||||
return x
|
||||
|
||||
# Copy of torchvision ShuffleNetV2 for type annotation
|
||||
def channel_shuffle(x, groups: int):
|
||||
batchsize, num_channels, height, width = x.data.size()
|
||||
channels_per_group = num_channels // groups
|
||||
|
||||
# reshape
|
||||
x = x.view(batchsize, groups,
|
||||
channels_per_group, height, width)
|
||||
|
||||
x = torch.transpose(x, 1, 2).contiguous()
|
||||
|
||||
# flatten
|
||||
x = x.view(batchsize, -1, height, width)
|
||||
|
||||
return x
|
||||
|
||||
class InvertedResidual(nn.Module):
|
||||
def __init__(self, inp, oup, stride):
|
||||
super(InvertedResidual, self).__init__()
|
||||
|
||||
if not (1 <= stride <= 3):
|
||||
raise ValueError('illegal stride value')
|
||||
self.stride = stride
|
||||
|
||||
branch_features = oup // 2
|
||||
assert (self.stride != 1) or (inp == branch_features << 1)
|
||||
|
||||
self.branch1 = nn.Sequential(
|
||||
self.depthwise_conv(inp, inp, kernel_size=3, stride=self.stride, padding=1),
|
||||
nn.BatchNorm2d(inp),
|
||||
nn.Conv2d(inp, branch_features, kernel_size=1, stride=1, padding=0, bias=False),
|
||||
nn.BatchNorm2d(branch_features),
|
||||
nn.ReLU(inplace=True),
|
||||
)
|
||||
|
||||
self.branch2 = nn.Sequential(
|
||||
nn.Conv2d(inp if (self.stride > 1) else branch_features,
|
||||
branch_features, kernel_size=1, stride=1, padding=0, bias=False),
|
||||
nn.BatchNorm2d(branch_features),
|
||||
nn.ReLU(inplace=True),
|
||||
self.depthwise_conv(branch_features, branch_features, kernel_size=3, stride=self.stride, padding=1),
|
||||
nn.BatchNorm2d(branch_features),
|
||||
nn.Conv2d(branch_features, branch_features, kernel_size=1, stride=1, padding=0, bias=False),
|
||||
nn.BatchNorm2d(branch_features),
|
||||
nn.ReLU(inplace=True),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def depthwise_conv(i, o, kernel_size, stride=1, padding=0, bias=False):
|
||||
return nn.Conv2d(i, o, kernel_size, stride, padding, bias=bias, groups=i)
|
||||
|
||||
def forward(self, x):
|
||||
if self.stride == 1:
|
||||
x1, x2 = x.chunk(2, dim=1)
|
||||
out = torch.cat((x1, self.branch2(x2)), dim=1)
|
||||
else:
|
||||
out = torch.cat((self.branch1(x), self.branch2(x)), dim=1)
|
||||
|
||||
out = channel_shuffle(out, 2)
|
||||
|
||||
return out
|
||||
|
||||
class ShuffleNetV2(nn.Module):
|
||||
def __init__(self, stages_repeats, stages_out_channels, num_classes=1000):
|
||||
super(ShuffleNetV2, self).__init__()
|
||||
|
||||
if len(stages_repeats) != 3:
|
||||
raise ValueError('expected stages_repeats as list of 3 positive ints')
|
||||
if len(stages_out_channels) != 5:
|
||||
raise ValueError('expected stages_out_channels as list of 5 positive ints')
|
||||
self._stage_out_channels = stages_out_channels
|
||||
|
||||
input_channels = 3
|
||||
output_channels = self._stage_out_channels[0]
|
||||
self.conv1 = nn.Sequential(
|
||||
nn.Conv2d(input_channels, output_channels, 3, 2, 1, bias=False),
|
||||
nn.BatchNorm2d(output_channels),
|
||||
nn.ReLU(inplace=True),
|
||||
)
|
||||
input_channels = output_channels
|
||||
|
||||
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
|
||||
|
||||
stage_names = ['stage{}'.format(i) for i in [2, 3, 4]]
|
||||
for name, repeats, output_channels in zip(
|
||||
stage_names, stages_repeats, self._stage_out_channels[1:]):
|
||||
seq = [InvertedResidual(input_channels, output_channels, 2)]
|
||||
for i in range(repeats - 1):
|
||||
seq.append(InvertedResidual(output_channels, output_channels, 1))
|
||||
setattr(self, name, nn.Sequential(*seq))
|
||||
input_channels = output_channels
|
||||
|
||||
output_channels = self._stage_out_channels[-1]
|
||||
self.conv5 = nn.Sequential(
|
||||
nn.Conv2d(input_channels, output_channels, 1, 1, 0, bias=False),
|
||||
nn.BatchNorm2d(output_channels),
|
||||
nn.ReLU(inplace=True),
|
||||
)
|
||||
|
||||
self.fc = nn.Linear(output_channels, num_classes)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.conv1(x)
|
||||
x = self.maxpool(x)
|
||||
x = self.stage2(x)
|
||||
x = self.stage3(x)
|
||||
x = self.stage4(x)
|
||||
x = self.conv5(x)
|
||||
x = x.mean([2, 3]) # globalpool
|
||||
x = self.fc(x)
|
||||
return x
|
||||
|
||||
# Facial landmark detection model
|
||||
class OpenSeeNetSNV2(ShuffleNetV2):
|
||||
def __init__(self):
|
||||
super(OpenSeeNetSNV2, self).__init__([4, 8, 4], [24, 116, 232, 464, 1024])
|
||||
self.up1 = UNetUp(1024, 232, 256, (14,14))
|
||||
self.up2 = UNetUp(256, 116, 198 * 1, (28,28))
|
||||
self.group = DSConv2d(198 * 1, 198, kernels_per_layer=4, groups=3)
|
||||
|
||||
def _forward_impl(self, x):
|
||||
x = self.conv1(x)
|
||||
x = self.maxpool(x)
|
||||
x = self.stage2(x)
|
||||
r2 = x
|
||||
x = self.stage3(x)
|
||||
r3 = x
|
||||
x = self.stage4(x)
|
||||
x = self.conv5(x)
|
||||
|
||||
x = self.up1(x, r3)
|
||||
x = self.up2(x, r2)
|
||||
x = self.group(x)
|
||||
return x
|
||||
|
||||
def forward(self, x):
|
||||
return self._forward_impl(x)
|
||||
|
||||
# Adaptive Wing Loss with offset layers and emphasis for eyes and eyebrows with 66 landmark points
|
||||
def AdapWingLoss(pre_hm, gt_hm):
|
||||
# pre_hm = pre_hm.to('cpu')
|
||||
# gt_hm = gt_hm.to('cpu')
|
||||
theta = 0.5
|
||||
alpha = 2.1
|
||||
w = 14
|
||||
e = 1
|
||||
A = w * (1 / (1 + torch.pow(theta / e, alpha - gt_hm))) * (alpha - gt_hm) * torch.pow(theta / e, alpha - gt_hm - 1) * (1 / e)
|
||||
C = (theta * A - w * torch.log(1 + torch.pow(theta / e, alpha - gt_hm)))
|
||||
|
||||
batch_size = gt_hm.size()[0]
|
||||
hm_num = gt_hm.size()[1]
|
||||
|
||||
mask = torch.zeros_like(gt_hm)
|
||||
# W = 10
|
||||
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))
|
||||
first = True
|
||||
first_mask = None
|
||||
for i in range(batch_size):
|
||||
img_list = []
|
||||
for j in range(hm_num // 3):
|
||||
hm = np.round(gt_hm[i][j].cpu().numpy() * 255)
|
||||
img_list.append(hm)
|
||||
img_merge = cv2.merge(img_list)
|
||||
img_dilate = cv2.morphologyEx(img_merge, cv2.MORPH_DILATE, kernel)
|
||||
img_dilate[img_dilate < 51] = 1 # 0*W+1
|
||||
img_dilate[img_dilate >= 51] = 11 # 1*W+1
|
||||
img_dilate = np.array(img_dilate, dtype=np.int)
|
||||
img_dilate = img_dilate.transpose(2, 0, 1)
|
||||
dilated = torch.from_numpy(img_dilate).float()
|
||||
if first:
|
||||
first_mask = dilated
|
||||
first = False
|
||||
dilated[17:27] *= 1.2
|
||||
dilated[17] *= 1.3
|
||||
dilated[18] *= 1.4
|
||||
dilated[25] *= 1.4
|
||||
dilated[26] *= 1.3
|
||||
dilated[36:47] *= 2.5
|
||||
mask[i] = torch.cat([dilated, dilated, dilated], 0)
|
||||
|
||||
diff_hm = torch.abs(gt_hm - pre_hm)
|
||||
AWingLoss = A * diff_hm - C
|
||||
idx = diff_hm < theta
|
||||
AWingLoss[idx] = w * torch.log(1 + torch.pow(diff_hm / e, alpha - gt_hm))[idx]
|
||||
|
||||
AWingLoss *= mask
|
||||
sum_loss = torch.sum(AWingLoss)
|
||||
all_pixel = torch.sum(mask)
|
||||
mean_loss = sum_loss / all_pixel
|
||||
|
||||
return first_mask.detach(), mean_loss
|
||||
|
||||
# The fast model runs on grayscale 112x112 using the ShuffleNet V2 0.5x configuration with input_channels=1. It was trained using AWL, but without the additional weighting for certain features
|
||||
.
|
||||
20719
models/haarcascade_frontalface_alt2.xml
Normal file
20719
models/haarcascade_frontalface_alt2.xml
Normal file
File diff suppressed because it is too large
Load diff
BIN
models/snv2_opt.onnx
Normal file
BIN
models/snv2_opt.onnx
Normal file
Binary file not shown.
BIN
models/snv2_opt_b.onnx
Normal file
BIN
models/snv2_opt_b.onnx
Normal file
Binary file not shown.
BIN
models/snv2_opt_fast.onnx
Normal file
BIN
models/snv2_opt_fast.onnx
Normal file
Binary file not shown.
474
tracker.py
Normal file
474
tracker.py
Normal file
|
|
@ -0,0 +1,474 @@
|
|||
import os
|
||||
import numpy as np
|
||||
import cv2
|
||||
import onnxruntime
|
||||
import time
|
||||
import queue
|
||||
import threading
|
||||
|
||||
def resolve(name):
|
||||
f = os.path.join(os.path.dirname(__file__), name)
|
||||
return f
|
||||
|
||||
def clamp_to_im(pt, w, h):
|
||||
x = pt[0]
|
||||
y = pt[1]
|
||||
if x < 0:
|
||||
x = 0
|
||||
if y < 0:
|
||||
y = 0
|
||||
if x >= w:
|
||||
x = w-1
|
||||
if y >= h:
|
||||
y = h-1
|
||||
return (int(x), int(y+1))
|
||||
|
||||
def intersects(r1, r2):
|
||||
r1_x1, r1_y1, r1_x2, r1_y2 = r1
|
||||
r1_x2 += r1_x1
|
||||
r1_y2 += r1_y1
|
||||
r2_x1, r2_y1, r2_x2, r2_y2 = r2
|
||||
r2_x2 += r2_x1
|
||||
r2_y2 += r2_y1
|
||||
return not (r1_x1 > r2_x2 or r1_x2 < r2_x1 or r1_y1 > r2_y2 or r1_y2 < r2_y1)
|
||||
|
||||
def group_rects(rects):
|
||||
rect_groups = {}
|
||||
for rect in rects:
|
||||
rect_groups[str(rect)] = [-1, -1, []]
|
||||
group_id = 0
|
||||
for i, rect in enumerate(rects):
|
||||
name = str(rect)
|
||||
group = group_id
|
||||
group_id += 1
|
||||
if rect_groups[name][0] < 0:
|
||||
rect_groups[name] = [group, -1, []]
|
||||
else:
|
||||
group = rect_groups[name][0]
|
||||
for j, other_rect in enumerate(rects):
|
||||
if i == j:
|
||||
continue;
|
||||
inter = intersects(rect, other_rect)
|
||||
if intersects(rect, other_rect):
|
||||
rect_groups[str(other_rect)] = [group, -1, []]
|
||||
return rect_groups
|
||||
|
||||
def logit(p):
|
||||
if p >= 1.0:
|
||||
p = 0.9999999
|
||||
if p <= 0.0:
|
||||
p = 0.0000001
|
||||
p = p/(1-p)
|
||||
return float(np.log(p) / 16.0)
|
||||
|
||||
def matrix_to_quaternion(m):
|
||||
t = 0.0
|
||||
q = [0.0, 0.0, 0, 0.0]
|
||||
if m[2,2] < 0:
|
||||
if m[0,0] > m[1,1]:
|
||||
t = 1 + m[0,0] - m[1,1] - m[2,2]
|
||||
q = [t, m[0,1]+m[1,0], m[2,0]+m[0,2], m[1,2]-m[2,1]]
|
||||
else:
|
||||
t = 1 - m[0,0] + m[1,1] - m[2,2]
|
||||
q = [m[0,1]+m[1,0], t, m[1,2]+m[2,1], m[2,0]-m[0,2]]
|
||||
else:
|
||||
if m[0,0] < -m[1,1]:
|
||||
t = 1 - m[0,0] - m[1,1] + m[2,2]
|
||||
q = [m[2,0]+m[0,2], m[1,2]+m[2,1], t, m[0,1]-m[1,0]]
|
||||
else:
|
||||
t = 1 + m[0,0] + m[1,1] + m[2,2]
|
||||
q = [m[1,2]-m[2,1], m[2,0]-m[0,2], m[0,1]-m[1,0], t]
|
||||
q = np.array(q, np.float32) * 0.5 / np.sqrt(t)
|
||||
return q
|
||||
|
||||
def worker_thread(session, input, crop_info, queue, input_name, idx, tracker):
|
||||
output = session.run([], {input_name: input})[0]
|
||||
conf, lms = tracker.landmarks(output[0], crop_info)
|
||||
queue.put((session, conf, lms, crop_info, idx))
|
||||
|
||||
class Tracker():
|
||||
def __init__(self, width, height, model_type=1, threshold=0.4, max_faces=1, discard_after=5, scan_every=3, bbox_growth=0.0, max_threads=4, silent=False, pnp_quality=1, model_dir=None):
|
||||
options = onnxruntime.SessionOptions()
|
||||
options.inter_op_num_threads = 1
|
||||
options.intra_op_num_threads = max_threads
|
||||
options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
|
||||
options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
options.log_severity_level = 3
|
||||
self.model_type = model_type
|
||||
self.models = ["snv2_opt.onnx", "snv2_opt_b.onnx", "snv2_opt_fast.onnx"]
|
||||
model = self.models[self.model_type]
|
||||
model_base_path = resolve(os.path.join("models"))
|
||||
if model_dir is None:
|
||||
if not os.path.exists(model_base_path):
|
||||
model_base_path = resolve(os.path.join("..", "models"))
|
||||
else:
|
||||
model_base_path = model_dir
|
||||
|
||||
# Single face instance with multiple threads
|
||||
self.session = onnxruntime.InferenceSession(os.path.join(model_base_path, model), sess_options=options)
|
||||
|
||||
# Multiple faces with single threads
|
||||
self.sessions = []
|
||||
self.max_workers = min(max_threads, max_faces)
|
||||
extra_threads = max_threads % self.max_workers
|
||||
for i in range(self.max_workers):
|
||||
options = onnxruntime.SessionOptions()
|
||||
options.inter_op_num_threads = 1
|
||||
options.intra_op_num_threads = max_threads // self.max_workers
|
||||
if options.intra_op_num_threads < 1:
|
||||
options.intra_op_num_threads = 1
|
||||
elif i < extra_threads:
|
||||
options.intra_op_num_threads += 1
|
||||
options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
|
||||
options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
self.sessions.append(onnxruntime.InferenceSession(os.path.join(model_base_path, model), sess_options=options))
|
||||
self.input_name = self.session.get_inputs()[0].name
|
||||
|
||||
self.faceCascade = cv2.CascadeClassifier()
|
||||
self.faceCascade.load(os.path.join(model_base_path, "haarcascade_frontalface_alt2.xml"))
|
||||
self.faces = []
|
||||
|
||||
# Image normalization constants
|
||||
self.mean = np.float32(np.array([0.485, 0.456, 0.406]))
|
||||
self.std = np.float32(np.array([0.229, 0.224, 0.225]))
|
||||
self.mean = self.mean / self.std
|
||||
self.std = self.std * 255.0
|
||||
|
||||
# PnP solving
|
||||
self.high_quality_3d = False if pnp_quality == 0 else True
|
||||
self.rotation = None
|
||||
self.translation = None
|
||||
self.face_3d = np.array([
|
||||
[0.50194, 0.28030, -0.49561],
|
||||
[0.48377, 0.15031, -0.50000],
|
||||
[0.46233, 0.01987, -0.49781],
|
||||
[0.43017, -0.11278, -0.49123],
|
||||
[0.38130, -0.25112, -0.46491],
|
||||
[0.31761, -0.35653, -0.40132],
|
||||
[0.24840, -0.44335, -0.30482],
|
||||
[0.15900, -0.51157, -0.22588],
|
||||
[0.00194, -0.55490, -0.16009],
|
||||
[-0.15900, -0.51157, -0.22588],
|
||||
[-0.24840, -0.44335, -0.30482],
|
||||
[-0.31761, -0.35653, -0.40132],
|
||||
[-0.38130, -0.25112, -0.46491],
|
||||
[-0.43017, -0.11278, -0.49123],
|
||||
[-0.46233, 0.01987, -0.49781],
|
||||
[-0.48377, 0.15031, -0.50000],
|
||||
[-0.50194, 0.28030, -0.49561],
|
||||
[0.44537, 0.38375, -0.22807],
|
||||
[0.38759, 0.42826, -0.18421],
|
||||
[0.32259, 0.44510, -0.12500],
|
||||
[0.24917, 0.44510, -0.07895],
|
||||
[0.19500, 0.42104, -0.06798],
|
||||
[-0.19500, 0.42104, -0.06798],
|
||||
[-0.24917, 0.44510, -0.07895],
|
||||
[-0.32259, 0.44510, -0.12500],
|
||||
[-0.38759, 0.42826, -0.18421],
|
||||
[-0.44537, 0.38375, -0.22807],
|
||||
[0.00000, 0.25503, -0.08553],
|
||||
[0.00000, 0.16240, -0.04825],
|
||||
[0.00000, 0.07218, -0.01535],
|
||||
[0.00000, 0.00000, 0.0000],
|
||||
[0.08353, -0.02165, -0.12281],
|
||||
[0.04738, -0.05019, -0.08991],
|
||||
[0.00000, -0.05895, -0.07456],
|
||||
[-0.04738, -0.05019, -0.08991],
|
||||
[-0.08353, -0.02165, -0.12281],
|
||||
[0.32981, 0.28270, -0.19956],
|
||||
[0.28528, 0.30435, -0.17544],
|
||||
[0.23111, 0.29714, -0.16009],
|
||||
[0.17213, 0.27789, -0.14474],
|
||||
[0.22870, 0.24781, -0.16009],
|
||||
[0.30093, 0.24661, -0.17544],
|
||||
[-0.17213, 0.27789, -0.14474],
|
||||
[-0.23111, 0.29714, -0.16009],
|
||||
[-0.28528, 0.30435, -0.17544],
|
||||
[-0.32981, 0.28270, -0.19956],
|
||||
[-0.30093, 0.24661, -0.17544],
|
||||
[-0.22870, 0.24781, -0.16009],
|
||||
[0.10871, -0.14282, -0.16667],
|
||||
[0.04809, -0.12118, -0.12500],
|
||||
[0.00000, -0.14931, -0.10307],
|
||||
[-0.04809, -0.12118, -0.12500],
|
||||
[-0.10871, -0.14282, -0.16667],
|
||||
[-0.11348, -0.24018, -0.16228],
|
||||
[-0.05719, -0.28129, -0.14474],
|
||||
[0.00000, -0.29644, -0.10965],
|
||||
[0.05719, -0.28129, -0.14474],
|
||||
[0.11348, -0.24018, -0.16228],
|
||||
[0.14984, -0.16878, -0.19956],
|
||||
[0.06736, -0.17922, -0.14254],
|
||||
[0.00000, -0.18861, -0.10965],
|
||||
[-0.06736, -0.17922, -0.14254],
|
||||
[-0.14984, -0.16878, -0.19956],
|
||||
[-0.04980, -0.21723, -0.14035],
|
||||
[0.00000, -0.21201, -0.11404],
|
||||
[0.04980, -0.21723, -0.14035],
|
||||
], np.float32) * np.array([1.0, 1.0, 1.15])
|
||||
|
||||
self.contour = np.empty((21 if self.high_quality_3d else 18, 3))
|
||||
self.contour[0:17] = self.face_3d[0:17]
|
||||
if self.high_quality_3d:
|
||||
self.contour[17:21] = self.face_3d[27:31]
|
||||
else:
|
||||
self.contour[17] = self.face_3d[30]
|
||||
self.camera = np.array([[width, 0, width/2], [0, width, height/2], [0, 0, 1]], np.float32)
|
||||
self.inverse_camera = np.linalg.inv(self.camera)
|
||||
self.dist_coeffs = np.zeros((4,1))
|
||||
|
||||
self.width = width
|
||||
self.height = height
|
||||
self.threshold = threshold
|
||||
self.max_faces = max_faces
|
||||
self.max_threads = max_threads
|
||||
self.discard = 0
|
||||
self.discard_after = discard_after
|
||||
self.detected = 0
|
||||
self.wait_count = 0
|
||||
self.scan_every = scan_every
|
||||
self.bbox_growth = bbox_growth
|
||||
self.silent = silent
|
||||
self.res = 224. if self.model_type < 2 else 112.
|
||||
self.res_i = int(self.res)
|
||||
|
||||
def landmarks(self, tensor, crop_info):
|
||||
crop_x1, crop_y1, scale_x, scale_y = crop_info
|
||||
avg_conf = 0
|
||||
lms = []
|
||||
for i in range(0, 66):
|
||||
m = int(tensor[i].argmax())
|
||||
x = m // 28
|
||||
y = m % 28
|
||||
conf = float(tensor[i][x,y])
|
||||
avg_conf = avg_conf + conf
|
||||
off_x = 0
|
||||
off_y = 0
|
||||
if self.model_type > 0:
|
||||
off_x = self.res * ((1. * logit(tensor[66 + i][y, x])) - 0.0)
|
||||
off_y = self.res * ((1. * logit(tensor[66 * 2 + i][y, x])) - 0.0)
|
||||
else:
|
||||
off_x = self.res * ((2. * float(tensor[66 + i][y, x])) - 1.0)
|
||||
off_y = self.res * ((2. * float(tensor[66 * 2 + i][y, x])) - 1.0)
|
||||
x = crop_y1 + scale_y * (self.res * (float(x) / 28.) + off_x)
|
||||
y = crop_x1 + scale_x * (self.res * (float(y) / 28.) + off_y)
|
||||
lms.append((x,y,conf))
|
||||
avg_conf = avg_conf / 66.
|
||||
return (avg_conf, lms)
|
||||
|
||||
def estimateDepth(self, lms):
|
||||
image_pts = np.empty((21 if self.high_quality_3d else 18, 2), np.float32)
|
||||
image_pts[0:17] = np.array(lms)[0:17,0:2]
|
||||
if (self.high_quality_3d):
|
||||
image_pts[17:21] = np.array(lms)[27:31,0:2]
|
||||
else:
|
||||
image_pts[17] = np.array(lms)[30,0:2]
|
||||
|
||||
success = False
|
||||
if self.max_faces < 2 and not self.rotation is None:
|
||||
success, self.rotation, self.translation = cv2.solvePnP(self.contour, image_pts, self.camera, self.dist_coeffs, useExtrinsicGuess=True, rvec=np.transpose(self.rotation), tvec=np.transpose(self.translation), flags=cv2.SOLVEPNP_EPNP)
|
||||
else:
|
||||
success, self.rotation, self.translation = cv2.solvePnP(self.contour, image_pts, self.camera, self.dist_coeffs, flags=cv2.SOLVEPNP_EPNP)
|
||||
|
||||
rotation = self.rotation
|
||||
translation = self.translation
|
||||
|
||||
if not success:
|
||||
self.rotation = np.array([0.0, 0.0, 0.0], np.float32)
|
||||
self.translation = np.array([0.0, 0.0, 0.0], np.float32)
|
||||
return False, np.zeros(4), 0.0, []
|
||||
else:
|
||||
self.rotation = np.transpose(self.rotation)
|
||||
self.translation = np.transpose(self.translation)
|
||||
|
||||
rmat, _ = cv2.Rodrigues(rotation)
|
||||
inverse_rotation = np.linalg.inv(rmat)
|
||||
pts_3d = []
|
||||
pnp_error = 0.0
|
||||
for i, pt in enumerate(self.face_3d):
|
||||
reference = rmat.dot(pt)
|
||||
reference = reference + self.translation
|
||||
reference = self.camera.dot(reference[0])
|
||||
depth = reference[2]
|
||||
if i < 17 or i == 30:
|
||||
reference = reference / depth
|
||||
e1 = lms[i][0] - reference[0]
|
||||
e2 = lms[i][1] - reference[1]
|
||||
pnp_error += e1*e1 + e2*e2
|
||||
pt_2d = np.array([lms[i][0] * depth, lms[i][1] * depth, depth], np.float32)
|
||||
pt_2d = self.inverse_camera.dot(pt_2d)
|
||||
pt_2d = pt_2d - self.translation
|
||||
pt_2d = inverse_rotation.dot(pt_2d[0])
|
||||
pts_3d.append(pt_2d)
|
||||
|
||||
euler = cv2.RQDecomp3x3(rmat)[0]
|
||||
return True, matrix_to_quaternion(rmat), euler, np.sqrt(pnp_error / (2 * 17.0)), pts_3d
|
||||
|
||||
def preprocess(self, im, crop):
|
||||
x1, y1, x2, y2 = crop
|
||||
im = np.float32(im[y1:y2, x1:x2,::-1]) # Crop and BGR to RGB
|
||||
im = cv2.resize(im, (self.res_i, self.res_i), interpolation=cv2.INTER_LINEAR) / self.std - self.mean
|
||||
#im = cv2.resize(im, (224, 224), interpolation=cv2.INTER_LINEAR) / 255.0
|
||||
#im = (im - mean) / std
|
||||
im = np.expand_dims(im, 0)
|
||||
im = np.transpose(im, (0,3,1,2))
|
||||
if self.model_type > 1:
|
||||
im = im.mean(1)
|
||||
im = np.expand_dims(im, 1)
|
||||
return im
|
||||
|
||||
def predict(self, frame, additional_faces=[]):
|
||||
start = time.time()
|
||||
im = frame
|
||||
|
||||
duration_fd = 0.0
|
||||
duration_pp = 0.0
|
||||
duration_model = 0.0
|
||||
duration_pnp = 0.0
|
||||
|
||||
new_faces = []
|
||||
new_faces.extend(self.faces)
|
||||
new_faces.extend(additional_faces)
|
||||
self.wait_count += 1
|
||||
if self.detected < self.max_faces:
|
||||
if self.detected == 0 or self.wait_count >= self.scan_every:
|
||||
start_fd = time.time()
|
||||
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
||||
new_faces.extend(list(self.faceCascade.detectMultiScale(gray, 1.3, 4, 0 | cv2.CASCADE_SCALE_IMAGE, (50, 50))))
|
||||
duration_fd = 1000 * (time.time() - start_fd)
|
||||
self.wait_count = 0
|
||||
else:
|
||||
self.wait_count = 0
|
||||
|
||||
if len(new_faces) < 1:
|
||||
duration = (time.time() - start) * 1000
|
||||
if not self.silent:
|
||||
print(f"Took {duration:.2f}ms")
|
||||
return []
|
||||
|
||||
crops = []
|
||||
crop_info = []
|
||||
num_crops = 0
|
||||
for j, (x,y,w,h) in enumerate(new_faces):
|
||||
if self.model_type < 2:
|
||||
crop_x1 = x - int(w * 0.2)
|
||||
crop_y1 = y - int(h * 0.25)
|
||||
crop_x2 = x + w + int(w * 0.2)
|
||||
crop_y2 = y + h + int(h * 0.25)
|
||||
else:
|
||||
crop_x1 = x - int(w * 0.1)
|
||||
crop_y1 = y - int(h * 0.1)
|
||||
crop_x2 = x + w + int(w * 0.1)
|
||||
crop_y2 = y + h + int(h * 0.1)
|
||||
crop_x1, crop_y1 = clamp_to_im((crop_x1, crop_y1), self.width, self.height)
|
||||
crop_x2, crop_y2 = clamp_to_im((crop_x2, crop_y2), self.width, self.height)
|
||||
|
||||
scale_x = float(crop_x2 - crop_x1) / self.res
|
||||
scale_y = float(crop_y2 - crop_y1) / self.res
|
||||
|
||||
if crop_x2 - crop_x1 < 4 or crop_y2 - crop_y1 < 4:
|
||||
continue
|
||||
|
||||
start_pp = time.time()
|
||||
crop = self.preprocess(im, (crop_x1, crop_y1, crop_x2, crop_y2))
|
||||
duration_pp += 1000 * (time.time() - start_pp)
|
||||
crops.append(crop)
|
||||
crop_info.append((crop_x1, crop_y1, scale_x, scale_y))
|
||||
num_crops += 1
|
||||
|
||||
groups = group_rects(new_faces)
|
||||
|
||||
start_model = time.time()
|
||||
outputs = {}
|
||||
if num_crops == 1:
|
||||
output = self.session.run([], {self.input_name: crops[0]})[0]
|
||||
conf, lms = self.landmarks(output[0], crop_info[0])
|
||||
outputs[crop_info[0]] = (conf, lms, 0)
|
||||
else:
|
||||
started = 0
|
||||
results = queue.Queue()
|
||||
for i in range(min(num_crops, self.max_workers)):
|
||||
thread = threading.Thread(target=worker_thread, args=(self.sessions[started], crops[started], crop_info[started], results, self.input_name, started, self))
|
||||
started += 1
|
||||
thread.start()
|
||||
returned = 0
|
||||
while returned < num_crops:
|
||||
session, conf, lms, sample_crop_info, idx = results.get(True)
|
||||
outputs[sample_crop_info] = (conf, lms, idx)
|
||||
returned += 1
|
||||
if started < num_crops:
|
||||
thread = threading.Thread(target=worker_thread, args=(session, crops[started], crop_info[started], results, self.input_name, started, self))
|
||||
started += 1
|
||||
thread.start()
|
||||
|
||||
best_results = {}
|
||||
for crop in crop_info:
|
||||
conf, lms, i = outputs[crop]
|
||||
if conf < self.threshold:
|
||||
continue;
|
||||
group_id = groups[str(new_faces[i])][0]
|
||||
if not group_id in best_results:
|
||||
best_results[group_id] = [-1, []]
|
||||
if conf > self.threshold and best_results[group_id][0] < conf:
|
||||
best_results[group_id][0] = conf
|
||||
best_results[group_id][1] = lms
|
||||
|
||||
sorted_results = sorted(best_results.values(), key=lambda x: x[0], reverse=True)
|
||||
duration_model = 1000 * (time.time() - start_model)
|
||||
|
||||
results = []
|
||||
detected = []
|
||||
detections = 0
|
||||
for conf, lms in sorted_results:
|
||||
if detections >= self.max_faces:
|
||||
break;
|
||||
if conf > self.threshold:
|
||||
detections += 1
|
||||
start_pnp = time.time()
|
||||
success, quaternion, euler, pnp_error, pts_3d = self.estimateDepth(lms)
|
||||
duration_pnp += 1000 * (time.time() - start_pnp)
|
||||
results.append((conf, lms, success, pnp_error, quaternion, euler, self.rotation, self.translation, pts_3d))
|
||||
min_x = 10000
|
||||
max_x = -1
|
||||
min_y = 10000
|
||||
max_y = -1
|
||||
for (x,y,c) in lms:
|
||||
if x > max_x:
|
||||
max_x = x
|
||||
if y > max_y:
|
||||
max_y = y
|
||||
if x < min_x:
|
||||
min_x = x
|
||||
if y < min_y:
|
||||
min_y = y
|
||||
w = max_x - min_x
|
||||
y = max_y - min_y
|
||||
detected.append((min_y, min_x, max_y - min_y, max_x - min_x))
|
||||
|
||||
if len(detected) > 0:
|
||||
self.detected = len(detected)
|
||||
self.faces = detected
|
||||
self.discard = 0
|
||||
else:
|
||||
self.rotation = None
|
||||
self.translation = None
|
||||
self.detected = 0
|
||||
self.discard += 1
|
||||
if self.discard > self.discard_after:
|
||||
self.faces = []
|
||||
else:
|
||||
if self.bbox_growth > 0:
|
||||
faces = []
|
||||
for (x,y,w,h) in self.faces:
|
||||
x -= w * self.bbox_growth
|
||||
y -= h * self.bbox_growth
|
||||
w += 2 * w * self.bbox_growth
|
||||
h += 2 * h * self.bbox_growth
|
||||
faces.append((x,y,w,h))
|
||||
self.faces = faces
|
||||
|
||||
duration = (time.time() - start) * 1000
|
||||
if not self.silent:
|
||||
print(f"Took {duration:.2f}ms (detect: {duration_fd:.2f}ms, crop: {duration_pp:.2f}, track: {duration_model:.2f}ms, 3D points: {duration_pnp:.2f}ms)")
|
||||
|
||||
return results
|
||||
Loading…
Add table
Add a link
Reference in a new issue