copilot/OpenSeeFace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

New expression detection code

Browse source
This commit is contained in:
Emiliana 2020-02-08 21:08:21 +01:00
parent 83e400a1d1
commit 3a83d96827
16 changed files with 4734 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

31
Licenses/libsvm.txt Normal file
View file

@ -0,0 +1,31 @@
Copyright (c) 2000-2019 Chih-Chung Chang and Chih-Jen Lin
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither name of copyright holders nor the names of its contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

192
Source/SVMWrapper.cpp Normal file
View file

@ -0,0 +1,192 @@
// SVMWrapper.cpp : Defines the exported functions for the DLL application.
//
#include "stdafx.h"
#include <cmath>
#include "svm.h"
using namespace std;
extern "C" void __cdecl print_log(const char *str) {
return;
}
struct SVMWrapper {
struct svm_model *model;
int cols;
double *means;
double *sdevs;
struct svm_parameter param;
struct svm_problem *problem;
};
typedef struct svm_node svmnode;
svmnode **make_svmnode_array(float features[], int rows, int cols, double *means, double *sdevs, int rescale) {
svmnode **x = new svmnode*[rows];
svmnode *nodes = new svmnode[rows * (cols + 1)];
double row_factor = 1.0 / (double)rows;
if (rescale) {
for (int c = 0; c < cols; c++) {
means[c] = 0;
sdevs[c] = 0;
}
for (int r = 0; r < rows; r++) {
for (int c = 0; c < cols; c++) {
means[c] += (double)features[r * cols + c];
}
}
for (int c = 0; c < cols; c++) {
means[c] *= row_factor;
}
for (int r = 0; r < rows; r++) {
for (int c = 0; c < cols; c++) {
double sq_diff = ((double)features[r * cols + c] - means[c]);
sq_diff *= sq_diff;
sdevs[c] += sq_diff;
}
}
for (int c = 0; c < cols; c++) {
sdevs[c] *= row_factor;
sdevs[c] = sqrt(sdevs[c]);
sdevs[c] = 1.0 / sdevs[c];
}
}
for (int r = 0; r < rows; r++) {
x[r] = &(nodes[r * (cols + 1)]);
int idx = 0;
for (int c = 0; c < cols; c++) {
x[r][c].index = c + 1;
x[r][c].value = ((double)features[r * cols + c] - means[c]) * sdevs[c];
}
x[r][cols].index = -1;
}
return x;
}
void make_problem(struct svm_problem *problem, float features[], float labels[], int rows, int cols, double *means, double *sdevs) {
problem->l = rows;
problem->y = new double[rows];
for (int i = 0; i < rows; i++)
problem->y[i] = (double)labels[i];
problem->x = make_svmnode_array(features, rows, cols, means, sdevs, 1);
}
void destroy_svmnode_array(svmnode **x) {
delete x[0];
delete x;
}
void destroy_problem(struct svm_problem *problem) {
delete problem->y;
destroy_svmnode_array(problem->x);
}
extern "C" {
__declspec(dllexport) void *__cdecl trainModel(float features[], float labels[], int rows, int cols) {
if (!(rows > 0 && cols > 0))
return NULL;
svm_set_print_string_function(print_log);
struct SVMWrapper *sw = new SVMWrapper();
sw->cols = cols;
sw->means = new double[cols];
sw->sdevs = new double[cols];
sw->param.svm_type = C_SVC;
sw->param.kernel_type = RBF;
sw->param.degree = 3;
sw->param.gamma = 1.0 / (double)cols;
sw->param.coef0 = 0;
sw->param.nu = 0.5;
sw->param.cache_size = 100;
sw->param.C = 4;
sw->param.eps = 0.001;
sw->param.p = 0.1;
sw->param.shrinking = 1;
sw->param.probability = 0;
sw->param.nr_weight = 0;
sw->param.weight_label = NULL;
sw->param.weight = NULL;
sw->problem = new struct svm_problem();
make_problem(sw->problem, features, labels, rows, cols, sw->means, sw->sdevs);
sw->model = svm_train(sw->problem, &sw->param);
return sw;
}
__declspec(dllexport) void __cdecl predict(void *ptr, float features[], float predictions[], int rows) {
struct SVMWrapper *sw = (struct SVMWrapper *)ptr;
int cols = sw->cols;
svmnode **x = make_svmnode_array(features, rows, cols, sw->means, sw->sdevs, 0);
for (int r = 0; r < rows; r++)
predictions[r] = (float)svm_predict(sw->model, x[r]);
destroy_svmnode_array(x);
return;
}
__declspec(dllexport) void *__cdecl loadModel(char *filename, int cols, double means[], double sdevs[]) {
struct SVMWrapper *sw = new SVMWrapper();
sw->cols = cols;
sw->model = svm_load_model(filename);
sw->means = new double[cols];
sw->sdevs = new double[cols];
for (int c = 0; c < cols; c++) {
sw->means[c] = means[c];
sw->sdevs[c] = sdevs[c];
}
return sw;
}
__declspec(dllexport) void *__cdecl loadModelString(char *modelString, int cols, double means[], double sdevs[]) {
struct SVMWrapper *sw = new SVMWrapper();
sw->cols = cols;
sw->model = svm_load_model_string(modelString);
sw->means = new double[cols];
sw->sdevs = new double[cols];
for (int c = 0; c < cols; c++) {
sw->means[c] = means[c];
sw->sdevs[c] = sdevs[c];
}
sw->problem = NULL;
return sw;
}
__declspec(dllexport) void __cdecl getScales(void *ptr, double means[], double sdevs[]) {
struct SVMWrapper *sw = (struct SVMWrapper *)ptr;
int cols = sw->cols;
for (int c = 0; c < cols; c++) {
means[c] = sw->means[c];
sdevs[c] = sw->sdevs[c];
}
}
__declspec(dllexport) char *__cdecl getModel(void *ptr) {
struct SVMWrapper *sw = (struct SVMWrapper *)ptr;
return svm_save_model_string(sw->model);
}
__declspec(dllexport) int __cdecl saveModel(void *ptr, char *filename) {
struct SVMWrapper *sw = (struct SVMWrapper *)ptr;
return svm_save_model(filename, sw->model);
}
__declspec(dllexport) void __cdecl destroyModel(void *ptr) {
struct SVMWrapper *sw = (struct SVMWrapper *)ptr;
delete sw->means;
delete sw->sdevs;
svm_free_and_destroy_model(&sw->model);
if (sw->problem != NULL) {
destroy_problem(sw->problem);
delete sw->problem;
}
delete sw;
}
}

3583
Source/svm.cpp Normal file
View file

File diff suppressed because it is too large Load diff

106
Source/svm.h Normal file
View file

@ -0,0 +1,106 @@
#ifndef _LIBSVM_H
#define _LIBSVM_H
#define LIBSVM_VERSION 324
#ifdef __cplusplus
extern "C" {
#endif
extern int libsvm_version;
struct svm_node
{
int index;
double value;
};
struct svm_problem
{
int l;
double *y;
struct svm_node **x;
};
enum { C_SVC, NU_SVC, ONE_CLASS, EPSILON_SVR, NU_SVR }; /* svm_type */
enum { LINEAR, POLY, RBF, SIGMOID, PRECOMPUTED }; /* kernel_type */
struct svm_parameter
{
int svm_type;
int kernel_type;
int degree; /* for poly */
double gamma; /* for poly/rbf/sigmoid */
double coef0; /* for poly/sigmoid */
/* these are for training only */
double cache_size; /* in MB */
double eps; /* stopping criteria */
double C; /* for C_SVC, EPSILON_SVR and NU_SVR */
int nr_weight; /* for C_SVC */
int *weight_label; /* for C_SVC */
double* weight; /* for C_SVC */
double nu; /* for NU_SVC, ONE_CLASS, and NU_SVR */
double p; /* for EPSILON_SVR */
int shrinking; /* use the shrinking heuristics */
int probability; /* do probability estimates */
};
//
// svm_model
//
struct svm_model
{
struct svm_parameter param; /* parameter */
int nr_class; /* number of classes, = 2 in regression/one class svm */
int l; /* total #SV */
struct svm_node **SV; /* SVs (SV[l]) */
double **sv_coef; /* coefficients for SVs in decision functions (sv_coef[k-1][l]) */
double *rho; /* constants in decision functions (rho[k*(k-1)/2]) */
double *probA; /* pariwise probability information */
double *probB;
int *sv_indices; /* sv_indices[0,...,nSV-1] are values in [1,...,num_traning_data] to indicate SVs in the training set */
/* for classification only */
int *label; /* label of each class (label[k]) */
int *nSV; /* number of SVs for each class (nSV[k]) */
/* nSV[0] + nSV[1] + ... + nSV[k-1] = l */
/* XXX */
int free_sv; /* 1 if svm_model is created by svm_load_model*/
/* 0 if svm_model is created by svm_train */
};
struct svm_model *svm_train(const struct svm_problem *prob, const struct svm_parameter *param);
void svm_cross_validation(const struct svm_problem *prob, const struct svm_parameter *param, int nr_fold, double *target);
int svm_save_model(const char *model_file_name, const struct svm_model *model);
char *svm_save_model_string(const svm_model *model);
struct svm_model *svm_load_model(const char *model_file_name);
struct svm_model *svm_load_model_string(char *model_string);
int svm_get_svm_type(const struct svm_model *model);
int svm_get_nr_class(const struct svm_model *model);
void svm_get_labels(const struct svm_model *model, int *label);
void svm_get_sv_indices(const struct svm_model *model, int *sv_indices);
int svm_get_nr_sv(const struct svm_model *model);
double svm_get_svr_probability(const struct svm_model *model);
double svm_predict_values(const struct svm_model *model, const struct svm_node *x, double* dec_values);
double svm_predict(const struct svm_model *model, const struct svm_node *x);
double svm_predict_probability(const struct svm_model *model, const struct svm_node *x, double* prob_estimates);
void svm_free_model_content(struct svm_model *model_ptr);
void svm_free_and_destroy_model(struct svm_model **model_ptr_ptr);
void svm_destroy_param(struct svm_parameter *param);
const char *svm_check_parameter(const struct svm_problem *prob, const struct svm_parameter *param);
int svm_check_probability_model(const struct svm_model *model);
void svm_set_print_string_function(void (*print_func)(const char *));
#ifdef __cplusplus
}
#endif
#endif /* _LIBSVM_H */

4
Unity/OpenSee.cs
View file

@ -16,7 +16,7 @@ public class OpenSee : MonoBehaviour {
public int listenPort = 11573;
private const int nPoints = 66;
private const int packetFrameSize = 8 + 2 * 4 + 2 * 4 + 1 + 4 + 3 * 4 + 3 * 4 + 4 * 4 + 4 * 68 + 4 * 2 * 68 + 4 * 3 * 70;
private const int packetFrameSize = 8 + 4 + 2 * 4 + 2 * 4 + 1 + 4 + 3 * 4 + 3 * 4 + 4 * 4 + 4 * 68 + 4 * 2 * 68 + 4 * 3 * 70;
[Header("Tracking data")]
[Tooltip("This is an informational property that tells you how many packets have been received")]
@ -28,7 +28,7 @@ public class OpenSee : MonoBehaviour {
public class OpenSeeData {
[Tooltip("The time this tracking data was captured at.")]
public double time;
[Tooltip("This is the id of the tracked face. When tracking multiple faces, they might get reordered due to faces coming and going, but as long as tracking is not lost on a face, its id should stay the same.")]
[Tooltip("This is the id of the tracked face. When tracking multiple faces, they might get reordered due to faces coming and going, but as long as tracking is not lost on a face, its id should stay the same. Face ids depend only on the order of first detection and locations of the faces.")]
public int id;
[Tooltip("This field gives the resolution of the camera or video being tracked.")]
public Vector2 cameraResolution;

11
Unity/OpenSee.cs.meta Normal file
View file

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 7b5af009eeb86f9469d2be8970e5a0f4
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

406
Unity/OpenSeeExpression.cs Normal file
View file

@ -0,0 +1,406 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Runtime.Serialization;
using System.Runtime.Serialization.Formatters.Binary;
using UnityEngine;
namespace OpenSee {
public class OpenSeeExpression : MonoBehaviour
{
[Header("Settings")]
[Tooltip("This is the source of face tracking data. If it is not set, any OpenSee component on the current object is used.")]
public OpenSee openSee;
[Tooltip("This specifies which expression calibration data should be collected for.")]
public string calibrationExpression = "neutral";
[Tooltip("This specifies the id of the face for which data should be collected and predictions should be made. Face ids depend only on the order of first detection and locations of the faces.")]
public int faceId = 0;
[Tooltip("This keeps low numbers of misdetections from changing the expression, but introduces a bit of lag to expression changes. For example, a value of 1 means that a new expression has to be detected twice in a row, before the predicted expression changes.")]
public int expressionStabilizer = 1;
[Tooltip("Setting this to a value above 1 will record only every recordingSkip-th frame when capture calibration data. This can be useful with cameras that have a high framerate leading to only a short time for capturing varied expression information. Usually at least 20-30 seconds are required to capture a good variety of angles and expression variation, so this should be set accordingly when using the recording flag. At 1, no frames are skipped.")]
public int recordingSkip = 2;
[Tooltip("This is the filename used for loading and saving expression data using the load and save flags.")]
public string filename = "";
[Header("Toggles")]
[Tooltip("When enabled, calibration data will be collected from the given OpenSee component.")]
public bool recording = false;
[Tooltip("When enabled, the calibration data collected for the current expression is cleared and this flag is set back to false.")]
public bool clear = false;
[Tooltip("When enabled, a new prediction model is trained and this flag is set back to false. Only expressions for which percentRecorded is 100% will be included in training.")]
public bool train = false;
[Tooltip("When enabled and a model is loaded, the current expression will be predicted every frame.")]
public bool predict = false;
[Tooltip("When enabled, the expression data from the specified filename will be loaded and this flag is set back to false.")]
public bool load = false;
[Tooltip("When enabled, the expression data will be saved to the specified filename and this flag is set back to false.")]
public bool save = false;
[Header("Information")]
[Tooltip("This is the of expressions for which calibration data was collected. The maximum number is 25.")]
public int expressionNumber = 0;
[Tooltip("This is the percentage of necessary training data collected for the current expression.")]
public float percentRecorded = 0.0f;
[Tooltip("This shows whether a model is ready to predict expressions with.")]
public bool modelReady = false;
[Tooltip("This is the time the current expression was detected.")]
public float expressionTime = 0f;
[Tooltip("This is currently detected expression.")]
public string expression = "";
[Header("Training statistics")]
[Tooltip("This shows the accuracy result of the last training run.")]
public float accuracy = 0.0f;
[Tooltip("This is the confusion matrix for the last training run's validation set.")]
public int[,] confusionMatrix = null;
[Tooltip("This is a pretty printed string representation of the confusion matrix.")]
public string confusionMatrixString = "";
[Tooltip("This shows any accuracy warnings that resulted from the last training run.")]
public string[] warnings = null;
[Serializable]
private class OpenSeeExpressionRepresentation {
private Dictionary<string, List<float[]>> expressions;
private byte[] modelBytes = null;
private string[] classLabels = null;
static public void LoadSerialized(byte[] modelBytes, out Dictionary<string, List<float[]>> expressions, out SVMModel model, out string[] classLabels) {
IFormatter formatter = new BinaryFormatter();
MemoryStream memoryStream = new MemoryStream(modelBytes);
memoryStream.Position = 0;
OpenSeeExpressionRepresentation oser;
using (GZipStream gzipStream = new GZipStream(memoryStream, CompressionMode.Decompress)) {
oser = formatter.Deserialize(gzipStream) as OpenSeeExpressionRepresentation;
}
expressions = oser.expressions;
model = new SVMModel(oser.modelBytes);
classLabels = oser.classLabels;
}
static public byte[] ToSerialized(Dictionary<string, List<float[]>> expressions, SVMModel model, string[] classLabels) {
OpenSeeExpressionRepresentation oser = new OpenSeeExpressionRepresentation();
oser.expressions = expressions;
oser.modelBytes = model.SaveModel();
oser.classLabels = classLabels;
IFormatter formatter = new BinaryFormatter();
MemoryStream memoryStream = new MemoryStream();
using (GZipStream gzipStream = new GZipStream(memoryStream, CompressionMode.Compress)) {
formatter.Serialize(gzipStream, oser);
gzipStream.Flush();
}
return memoryStream.ToArray();
}
}
private Dictionary<string, List<float[]>> expressions;
private SVMModel model = null;
private string[] classLabels = null;
private int maxSamples = 400; // Allows 25 expressions without exceeding 10000 rows in total for SVM training, which is a commonly seen upper limit.
// rightEyeOpen, leftEyeOpen, translation, rawQuaternion, rawEuler, confidence, points/(width, height), points3D
private int cols = 1 + 1 + 3 + 4 + 3 + /*66 + 2 * 66 +*/ 3 * 66;
private double lastCapture = 0.0;
private float warningThreshold = 5;
private int currentPrediction = -1;
private int lastPrediction = -1;
private int lastPredictionCount = 0;
private int frameCount = 0;
private void ResetInfo() {
recording = false;
clear = false;
train = false;
predict = false;
modelReady = false;
accuracy = 0.0f;
confusionMatrix = null;
warnings = null;
percentRecorded = 0.0f;
expression = "";
confusionMatrixString = "";
lastPrediction = -1;
currentPrediction = -1;
lastPredictionCount = 0;
expressionTime = 0f;
}
void Start()
{
if (openSee == null)
openSee = GetComponent<OpenSee>();
if (openSee == null) {
ResetInfo();
return;
}
ResetInfo();
expressions = new Dictionary<string, List<float[]>>();
model = new SVMModel();
}
private float[] GetData(OpenSee.OpenSeeData t) {
float[] data = new float[cols];
data[0] = t.rightEyeOpen;
data[1] = t.leftEyeOpen;
data[2] = t.translation.x;
data[3] = t.translation.y;
data[4] = t.translation.z;
data[5] = t.rawQuaternion.x;
data[6] = t.rawQuaternion.y;
data[7] = t.rawQuaternion.z;
data[8] = t.rawQuaternion.w;
data[9] = t.rawEuler.x;
data[10] = t.rawEuler.y;
data[11] = t.rawEuler.z;
/*for (int i = 0; i < 66; i++)
data[12 + i] = t.confidence[i];
for (int i = 0; i < 66; i++) {
data[12 + 66 + 2 * i] = t.points[i].x / t.cameraResolution.x;
data[12 + 66 + 2 * i + 1] = t.points[i].y / t.cameraResolution.y;
}*/
for (int i = 0; i < 66; i++) {
data[12 + /*66 + 2 * 66 +*/ 3 * i] = t.points3D[i].x;
data[12 + /*66 + 2 * 66 +*/ 3 * i + 1] = t.points3D[i].y;
data[12 + /*66 + 2 * 66 +*/ 3 * i + 2] = t.points3D[i].z;
}
return data;
}
public bool TrainModel() {
train = false;
if (openSee == null) {
ResetInfo();
return false;
}
List<string> keys = new List<string>();
foreach (string key in expressions.Keys) {
List<float[]> list = expressions[key];
if (list.Count == maxSamples)
keys.Add(key);
else
return false;
}
int classes = keys.Count;
if (classes < 2 || classes > 25)
return false;
keys.Sort();
classLabels = keys.ToArray();
int train_split = maxSamples * 3 / 4;
int test_split = maxSamples - train_split;
int rows_train = classes * train_split;
int rows_test = classes * test_split;
float[] X_train = new float[rows_train * cols];
float[] X_test = new float[rows_test * cols];
float[] y_train = new float[rows_train];
float[] y_test = new float[rows_test];
int i_train = 0;
int i_test = 0;
System.Random rnd = new System.Random();
for (int i = 0; i < classes; i++) {
List<float[]> list = expressions[keys[i]];
for (int j = maxSamples; j > 1;) {
j--;
int k = rnd.Next(j + 1);
float[] tmp = list[k];
list[k] = list[j];
list[j]= tmp;
}
for (int j = 0; j < train_split; j++) {
for (int k = 0; k < cols; k++) {
X_train[i_train * cols + k] = list[j][k];
}
y_train[i_train] = i;
i_train++;
}
for (int j = train_split; j < train_split + test_split; j++) {
for (int k = 0; k < cols; k++)
X_test[i_test * cols + k] = list[j][k];
y_test[i_test] = i;
i_test++;
}
}
model.TrainModel(X_train, y_train, rows_train, cols);
confusionMatrix = model.ConfusionMatrix(X_test, y_test, rows_test, out accuracy);
confusionMatrixString = SVMModel.FormatMatrix(confusionMatrix, classLabels);
List<string> accuracyWarnings = new List<string>();
for (int label = 0; label < classes; label++) {
float error = 100f * (1f - ((float)confusionMatrix[label, label] / (float)test_split));
if (error > warningThreshold)
accuracyWarnings.Add("The expression \"" + classLabels[label] + "\" is misclassified with a chance of " + error.ToString("0.00") + "%.");
}
warnings = accuracyWarnings.ToArray();
modelReady = true;
return true;
}
public bool PredictExpression() {
if (openSee == null || !(modelReady && model.Ready())) {
ResetInfo();
return false;
}
OpenSee.OpenSeeData[] openSeeData = openSee.trackingData;
if (openSeeData == null)
return false;
OpenSee.OpenSeeData t = null;
foreach (OpenSee.OpenSeeData data in openSeeData)
if (data.id == faceId)
t = data;
if (t == null || t.time <= lastCapture)
return false;
float[] faceData = GetData(t);
float[] prediction = model.Predict(faceData, 1);
int predictedExpression = (int)Mathf.Round(prediction[0]);
if (predictedExpression == lastPrediction)
lastPredictionCount++;
else {
lastPrediction = predictedExpression;
lastPredictionCount = 1;
}
if (lastPredictionCount > expressionStabilizer) {
currentPrediction = lastPrediction;
expressionTime = Time.time;
}
if (currentPrediction >= 0 && currentPrediction <= classLabels.Length)
expression = classLabels[currentPrediction];
else
expression = "";
return true;
}
public bool CaptureCalibrationData() {
if (openSee == null) {
ResetInfo();
return false;
}
if (recordingSkip < 1)
recordingSkip = 1;
if (calibrationExpression == "")
return false;
if (expressions.ContainsKey(calibrationExpression) || expressions.Keys.Count < 25) {
OpenSee.OpenSeeData[] openSeeData = openSee.trackingData;
if (openSeeData == null)
return false;
OpenSee.OpenSeeData t = null;
foreach (OpenSee.OpenSeeData data in openSeeData)
if (data.id == faceId)
t = data;
if (t == null || t.time <= lastCapture)
return false;
if (!expressions.ContainsKey(calibrationExpression)) {
expressions.Add(calibrationExpression, new List<float[]>());
}
List<float[]> list = expressions[calibrationExpression];
if (list.Count >= maxSamples) {
percentRecorded = 100f;
recording = false;
return false;
}
if (frameCount % recordingSkip != 0)
return true;
list.Add(GetData(t));
percentRecorded = 100f * (float)list.Count/(float)maxSamples;
return true;
} else {
recording = false;
calibrationExpression = "";
return false;
}
}
public string[] GetExpressions() {
List<string> keys = new List<string>();
foreach (string key in expressions.Keys)
keys.Add(key);
keys.Sort();
return keys.ToArray();
}
public void ClearExpression() {
if (expressions.ContainsKey(calibrationExpression))
expressions.Remove(calibrationExpression);
clear = false;
}
public void LoadFromBytes(byte[] data) {
load = false;
if (openSee == null) {
ResetInfo();
return;
}
OpenSeeExpressionRepresentation.LoadSerialized(data, out expressions, out model, out classLabels);
if (model.Ready())
modelReady = true;
}
public byte[] SaveToBytes() {
save = false;
if (openSee == null) {
ResetInfo();
return null;
}
return OpenSeeExpressionRepresentation.ToSerialized(expressions, model, classLabels);
}
public void LoadFromFile() {
load = false;
if (openSee == null) {
ResetInfo();
return;
}
byte[] data = File.ReadAllBytes(filename);
LoadFromBytes(data);
}
public void SaveToFile() {
save = false;
if (openSee == null) {
ResetInfo();
return;
}
byte[] data = SaveToBytes();
File.WriteAllBytes(filename, data);
}
void Update()
{
frameCount++;
if (openSee == null || expressions == null) {
ResetInfo();
return;
}
if (clear)
ClearExpression();
if (recording && calibrationExpression != "") {
CaptureCalibrationData();
}
if (train)
TrainModel();
expressionNumber = expressions.Keys.Count;
if (load)
LoadFromFile();
if (save)
SaveToFile();
if (!predict)
expression = "";
else if (modelReady)
PredictExpression();
else
predict = false;
if (expressions.ContainsKey(calibrationExpression))
percentRecorded = 100f * (float)expressions[calibrationExpression].Count/(float)maxSamples;
else
percentRecorded = 0f;
OpenSee.OpenSeeData[] openSeeData = openSee.trackingData;
if (openSeeData == null)
return;
OpenSee.OpenSeeData t = null;
foreach (OpenSee.OpenSeeData data in openSeeData)
if (data.id == faceId)
t = data;
if (t == null || t.time <= lastCapture)
return;
lastCapture = t.time;
}
}
}

11
Unity/OpenSeeExpression.cs.meta Normal file
View file

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 9066de4c7793c1e40b212724f72fc064
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

11
Unity/OpenSeeJobManagement.cs.meta Normal file
View file

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 3251308a085a78545a82e7f34ebc2d34
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

11
Unity/OpenSeeLauncher.cs.meta Normal file
View file

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: beae834477f91834691fb7248fddacf6
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

11
Unity/OpenSeeShowPoints.cs.meta Normal file
View file

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 9c0b8bb971ed0e14288631f4274a3807
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

8
Unity/Plugins.meta Normal file
View file

@ -0,0 +1,8 @@
fileFormatVersion: 2
guid: f1dfc5f1c2e6f294e8191828595882b1
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

248
Unity/Plugins/SVMModel.cs Normal file
View file

@ -0,0 +1,248 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Runtime.Serialization.Formatters.Binary;
using System.Text;
namespace OpenSee {
public class SVMModel {
#region DllImport
[DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl)]
private static extern System.IntPtr trainModel(float[] features, float[] labels, int rows, int cols);
[DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl)]
private static extern void predict(System.IntPtr model, float[] features, [Out] float[] predictions, int rows);
[DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)]
private static extern System.IntPtr loadModel([MarshalAs(UnmanagedType.LPStr)]string filename, int cols, double[] means, double[] sdevs);
[DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)]
private static extern System.IntPtr loadModelString([MarshalAs(UnmanagedType.LPStr)]string modelString, int cols, double[] means, double[] sdevs);
[DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl)]
private static extern void getScales(System.IntPtr model, [Out] double[] means, [Out] double[] sdevs);
[DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)]
private static extern int saveModel(System.IntPtr model, [MarshalAs(UnmanagedType.LPStr)]string filename);
[DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)]
private static extern string saveModelString(System.IntPtr model);
[DllImport("SVMModel", CallingConvention = CallingConvention.Cdecl)]
private static extern void destroyModel(System.IntPtr model);
#endregion
private bool haveModel = false;
private System.IntPtr model;
private int cols;
private int maxClasses;
[Serializable]
private class SVMModelRepresentation {
private string modelString;
private int cols;
private int maxClasses;
private double[] means;
private double[] sdevs;
static public System.IntPtr LoadSerialized(byte[] modelBytes, out int cols, out int maxClasses, bool compress) {
IFormatter formatter = new BinaryFormatter();
MemoryStream memoryStream = new MemoryStream(modelBytes);
memoryStream.Position = 0;
SVMModelRepresentation smr;
if (compress) {
using (GZipStream gzipStream = new GZipStream(memoryStream, CompressionMode.Decompress)) {
smr = formatter.Deserialize(gzipStream) as SVMModelRepresentation;
}
} else {
smr = formatter.Deserialize(memoryStream) as SVMModelRepresentation;
}
cols = smr.cols;
maxClasses = smr.maxClasses;
System.IntPtr model = loadModelString(smr.modelString, smr.cols, smr.means, smr.sdevs);
return model;
}
static public byte[] ToSerialized(System.IntPtr model, int cols, int maxClasses, bool compress) {
SVMModelRepresentation smr = new SVMModelRepresentation();
smr.cols = cols;
smr.maxClasses = maxClasses;
smr.means = new double[cols];
smr.sdevs = new double[cols];
getScales(model, smr.means, smr.sdevs);
smr.modelString = saveModelString(model);
IFormatter formatter = new BinaryFormatter();
MemoryStream memoryStream = new MemoryStream();
if (compress) {
using (GZipStream gzipStream = new GZipStream(memoryStream, CompressionMode.Compress)) {
formatter.Serialize(gzipStream, smr);
gzipStream.Flush();
}
} else {
formatter.Serialize(memoryStream, smr);
}
return memoryStream.ToArray();
}
}
public void DestroyModel() {
if (haveModel) {
destroyModel(model);
haveModel = false;
}
}
private void LoadModel(byte[] modelBytes, bool compress) {
DestroyModel();
model = SVMModelRepresentation.LoadSerialized(modelBytes, out cols, out maxClasses, compress);
haveModel = true;
}
public void LoadModel(byte[] modelBytes) {
LoadModel(modelBytes, true);
}
private byte[] SaveModel(bool compress) {
if (!haveModel)
return null;
return SVMModelRepresentation.ToSerialized(model, cols, maxClasses, compress);
}
public byte[] SaveModel() {
return SaveModel(true);
}
// features is a rows*cols long float array with rows written one after another. labels is a rows long float array with the corresponding classes as integral numbers from 0 to num_classes-1, where all classes have to exist. Training on more than 10000 rows is disabled.
public bool TrainModel(float[] features, float[] labels, int rows, int cols) {
DestroyModel();
if (rows > 10000)
return false;
if (cols < 1 || rows < 1 || features.Length != rows * cols || labels.Length != rows)
return false;
int max = -1;
HashSet<int> knownClasses = new HashSet<int>();
for (int i = 0; i < labels.Length; i++) {
labels[i] = (float)Math.Round(labels[i]);
int l = (int)labels[i];
if (l > max)
max = l;
if (l < 0)
return false;
knownClasses.Add(l);
}
for (int i = 0; i <= max; i++) {
if (!knownClasses.Contains(i))
return false;
}
maxClasses = max + 1;
this.cols = cols;
model = trainModel(features, labels, rows, cols);
haveModel = true;
return true;
}
// This function takes the same arguments as train, except for cols, which is not necessary. The diagonal are correctly classified entries. The first index is labels, the second index is classifications.
public int[,] ConfusionMatrix(float[] features, float[] labels, int rows, out float accuracy) {
accuracy = 0f;
if (!haveModel)
return null;
if (rows < 1 || features.Length != rows * cols || labels.Length != rows)
return null;
for (int i = 0; i < labels.Length; i++) {
labels[i] = (float)Math.Round(labels[i]);
if (labels[i] < 0 || labels[i] >= maxClasses)
return null;
}
int[,] confusionMatrix = new int[maxClasses, maxClasses];
float[] predictions = new float[rows];
predict(model, features, predictions, rows);
for (int i = 0; i < rows; i++) {
predictions[i] = (float)Math.Round(predictions[i]);
if (predictions[i] >= 0 && predictions[i] < maxClasses) {
confusionMatrix[(int)labels[i], (int)predictions[i]]++;
if ((int)labels[i] == (int)predictions[i])
accuracy++;
}
}
accuracy /= (float)rows;
return confusionMatrix;
}
static public string FormatMatrix(int[,] matrix) {
return FormatMatrix(matrix, null);
}
static public string FormatMatrix(int[,] matrix, string[] labels) {
StringBuilder sb = new StringBuilder();
int rowLength = matrix.GetLength(0);
int colLength = matrix.GetLength(1);
int maxLableLen = 5;
if (labels != null) {
if (labels.Length != rowLength || labels.Length != colLength)
labels = null;
else {
foreach (string label in labels)
if (label.Length > maxLableLen)
maxLableLen = label.Length;
sb.Append("pred:".PadLeft(maxLableLen));
sb.Append(" ");
foreach (string label in labels) {
sb.Append(label.PadLeft(maxLableLen));
sb.Append(" ");
}
sb.Append(Environment.NewLine);
}
}
for (int r = 0; r < rowLength; r++) {
if (labels != null) {
sb.Append(labels[r].PadLeft(maxLableLen));
sb.Append(" ");
}
for (int c = 0; c < colLength; c++) {
string n = matrix[r,c].ToString();
sb.Append(n.PadLeft(maxLableLen));
sb.Append(" ");
}
sb.Append(Environment.NewLine);
}
return sb.ToString();
}
// features is a rows*cols long float array with rows written one after another.
public float[] Predict(float[] features, int rows) {
if (!haveModel)
return null;
if (rows < 1 || features.Length != rows * cols)
return null;
float[] predictions = new float[rows];
predict(model, features, predictions, rows);
return predictions;
}
public bool Ready() {
return haveModel;
}
public SVMModel() {
}
public SVMModel(byte[] modelBytes) {
LoadModel(modelBytes);
}
~SVMModel() {
DestroyModel();
}
}
}

11
Unity/Plugins/SVMModel.cs.meta Normal file
View file

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: a95b57bbe200b004dab9b21f0854082e
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

BIN
Unity/Plugins/SVMModel.dll Normal file
View file

Binary file not shown.

92
Unity/Plugins/SVMModel.dll.meta Normal file
View file

@ -0,0 +1,92 @@
fileFormatVersion: 2
guid: 930906668f1b67a42b2bb844bd098a26
PluginImporter:
externalObjects: {}
serializedVersion: 2
iconMap: {}
executionOrder: {}
isPreloaded: 0
isOverridable: 0
platformData:
- first:
'': Any
second:
enabled: 0
settings:
Exclude Editor: 0
Exclude Linux: 0
Exclude Linux64: 0
Exclude LinuxUniversal: 0
Exclude OSXUniversal: 0
Exclude WebGL: 0
Exclude Win: 1
Exclude Win64: 0
- first:
Any:
second:
enabled: 1
settings: {}
- first:
Editor: Editor
second:
enabled: 1
settings:
CPU: AnyCPU
DefaultValueInitialized: true
OS: AnyOS
- first:
Facebook: Win
second:
enabled: 0
settings:
CPU: None
- first:
Facebook: Win64
second:
enabled: 0
settings:
CPU: AnyCPU
- first:
Standalone: Linux
second:
enabled: 1
settings:
CPU: x86
- first:
Standalone: Linux64
second:
enabled: 1
settings:
CPU: x86_64
- first:
Standalone: LinuxUniversal
second:
enabled: 1
settings:
CPU: AnyCPU
- first:
Standalone: OSXUniversal
second:
enabled: 1
settings:
CPU: AnyCPU
- first:
Standalone: Win
second:
enabled: 0
settings:
CPU: None
- first:
Standalone: Win64
second:
enabled: 1
settings:
CPU: AnyCPU
- first:
WebGL: WebGL
second:
enabled: 1
settings: {}
userData:
assetBundleName:
assetBundleVariant: