First version.

Unity/OpenSee.cs

@@ -0,0 +1,170 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Net;
+using System.Net.Sockets;
+using System.Threading;
+using UnityEngine;
+
+namespace OpenSee {
+
+public class OpenSee : MonoBehaviour {
+    [Header("UDP server settings")]
+    [Tooltip("To listen for remote connections, change this to 0.0.0.0 or your actual IP on the desired interface")]
+    public string listenAddress = "127.0.0.1";
+    [Tooltip("This is the port the server will listen for tracking packets on")]
+    public int listenPort = 11573;
+
+    private const int nPoints = 66;
+    private const int packetFrameSize = 8 + 1 + 4 + 4 * 4 + 3 * 4 + 3 * 4 + 4 * 66 + 4 * 2 * 66 + 4 * 3 * 66;
+
+    [Header("Tracking data")]
+    [Tooltip("This is an informational property that tells you how many packets have been received")]
+    public int receivedPackets = 0;
+    [Tooltip("This contains the actual tracking data")]
+    public OpenSeeData[] trackingData = null;
+
+    [System.Serializable]
+    public class OpenSeeData {
+        [Tooltip("The time this tracking data was captured at.")]
+        public double time;
+        [Tooltip("This field tells you if 3D points have been successfully estimated from the 2D points. If this is false, do not rely on pose or 3D data.")]
+        public bool got3DPoints;
+        [Tooltip("This field contains the error for fitting the original 3D points. It shouldn't matter much, but it it is very high, something is probably wrong")]
+        public float fit3DError;
+        [Tooltip("This is the rotation vector for the 3D points to turn into the estimated face pose.")]
+        public Vector3 rotation;
+        [Tooltip("This is the translation vector for the 3D points to turn into the estimated face pose.")]
+        public Vector3 translation;
+        [Tooltip("This is the raw rotation quaternion calculated from the OpenCV rotation matrix. It does not match Unity's coordinate system, but it still might be useful.")]
+        public Quaternion rawQuaternion;
+        [Tooltip("This is the raw rotation euler angles calculated by OpenCV from the rotation matrix. It does not match Unity's coordinate system, but it still might be useful.")]
+        public Vector3 rawEuler;
+        [Tooltip("This field tells you how certain the tracker is.")]
+        public float[] confidence;
+        [Tooltip("These are the detected face landmarks in image coordinates.")]
+        public Vector2[] points;
+        [Tooltip("These are 3D points estimated from the 2D points. The should be rotation and translation compensated.")]
+        public Vector3[] points3D;
+
+        public OpenSeeData() {
+            confidence = new float[nPoints];
+            points = new Vector2[nPoints];
+            points3D = new Vector3[nPoints];
+        }
+
+        private float readFloat(byte[] b, ref int o) {
+            float v = System.BitConverter.ToSingle(b, o);
+            o += 4;
+            return v;
+        }
+
+        private Quaternion readQuaternion(byte[] b, ref int o) {
+            float x = readFloat(b, ref o);
+            float y = readFloat(b, ref o);
+            float z = readFloat(b, ref o);
+            float w = readFloat(b, ref o);
+            Quaternion q = new Quaternion(x, y, z, w);
+            return q;
+        }
+
+        private Vector3 readVector3(byte[] b, ref int o) {
+            Vector3 v = new Vector3(readFloat(b, ref o), -readFloat(b, ref o), readFloat(b, ref o));
+            return v;
+        }
+
+        private Vector2 readVector2(byte[] b, ref int o) {
+            Vector2 v = new Vector2(readFloat(b, ref o), readFloat(b, ref o));
+            return v;
+        }
+
+        public void readFromPacket(byte[] b, int o) {
+            time = System.BitConverter.ToDouble(b, o);
+            o += 8;
+
+            byte got3D = b[o];
+            o++;
+            got3DPoints = false;
+            if (got3D != 0)
+                got3DPoints = true;
+
+            fit3DError = readFloat(b, ref o);
+            rawQuaternion = readQuaternion(b, ref o);
+            Quaternion convertedQuaternion = new Quaternion(-rawQuaternion.x, rawQuaternion.y, -rawQuaternion.z, rawQuaternion.w);
+            rawEuler = readVector3(b, ref o);
+
+            rotation = rawEuler;
+            rotation.z = (rotation.z + 90) % 360;
+            rotation.x = -(rotation.x + 180) % 360;
+
+            float x = readFloat(b, ref o);
+            float y = readFloat(b, ref o);
+            float z = readFloat(b, ref o);
+            translation = new Vector3(y, -x, z);
+
+            for (int i = 0; i < nPoints; i++) {
+                confidence[i] = readFloat(b, ref o);
+            }
+
+            for (int i = 0; i < nPoints; i++) {
+                points[i] = readVector2(b, ref o);
+            }
+
+            for (int i = 0; i < nPoints; i++) {
+                points3D[i] = readVector3(b, ref o);
+            }
+        }
+    }
+
+    private Socket socket;
+    private byte[] buffer;
+    private Thread receiveThread = null;
+    private volatile bool stopReception = false;
+
+    void performReception() {
+        EndPoint senderRemote = new IPEndPoint(IPAddress.Any, 0);
+        while (!stopReception) {
+            try {
+                int receivedBytes = socket.ReceiveFrom(buffer, SocketFlags.None, ref senderRemote);
+                if (receivedBytes < 1 || receivedBytes % packetFrameSize != 0) {
+                    continue;
+                }
+                receivedPackets++;
+                int i = 0;
+                OpenSeeData[] newData = new OpenSeeData[receivedBytes / packetFrameSize];
+                for (int offset = 0; offset < receivedBytes; offset += packetFrameSize) {
+                    newData[i] = new OpenSeeData();
+                    newData[i].readFromPacket(buffer, offset);
+                    i++;
+                }
+                trackingData = newData;
+            } catch { }
+        }
+    }
+
+    void Start () {
+        buffer = new byte[65535];
+
+        socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
+        IPAddress ip;
+        IPAddress.TryParse(listenAddress, out ip);
+        socket.Bind(new IPEndPoint(ip, listenPort));
+        socket.ReceiveTimeout = 15;
+
+        receiveThread = new Thread(() => performReception());
+        receiveThread.Start();
+    }
+
+    void Update () {
+	}
+
+    void OnApplicationQuit() {
+        if (receiveThread != null) {
+            stopReception = true;
+            receiveThread.Join();
+            stopReception = false;
+        }
+    }
+}
+
+}

Unity/OpenSeeJobManagement.cs

@@ -0,0 +1,150 @@
+using System;
+using System.Diagnostics;
+using System.Runtime.InteropServices;
+
+// From: https://stackoverflow.com/a/9164742
+// Is this better? https://stackoverflow.com/a/36648201
+namespace OpenSee
+{
+    public class Win32 {
+        [DllImport("kernel32.dll", SetLastError = true)] [return: MarshalAs(UnmanagedType.Bool)]
+        static extern bool CloseHandle(IntPtr hObject);
+    }
+    
+    public class Job : IDisposable
+    {
+        [DllImport("kernel32.dll", CharSet = CharSet.Unicode)]
+        static extern IntPtr CreateJobObject(IntPtr a, string lpName);
+
+        [DllImport("kernel32.dll")]
+        static extern bool SetInformationJobObject(IntPtr hJob, JobObjectInfoType infoType, IntPtr lpJobObjectInfo, UInt32 cbJobObjectInfoLength);
+
+        [DllImport("kernel32.dll", SetLastError = true)]
+        static extern bool AssignProcessToJobObject(IntPtr job, IntPtr process);
+
+        [DllImport("kernel32.dll", SetLastError = true)]
+        [return: MarshalAs(UnmanagedType.Bool)]
+        static extern bool CloseHandle(IntPtr hObject);
+
+        private IntPtr handle;
+        private bool disposed;
+
+        public Job()
+        {
+            handle = CreateJobObject(IntPtr.Zero, null);
+
+            var info = new JOBOBJECT_BASIC_LIMIT_INFORMATION
+            {
+                LimitFlags = 0x2000
+            };
+
+            var extendedInfo = new JOBOBJECT_EXTENDED_LIMIT_INFORMATION
+            {
+                BasicLimitInformation = info
+            };
+
+            int length = Marshal.SizeOf(typeof(JOBOBJECT_EXTENDED_LIMIT_INFORMATION));
+            IntPtr extendedInfoPtr = Marshal.AllocHGlobal(length);
+            Marshal.StructureToPtr(extendedInfo, extendedInfoPtr, false);
+
+            if (!SetInformationJobObject(handle, JobObjectInfoType.ExtendedLimitInformation, extendedInfoPtr, (uint)length))
+                throw new Exception(string.Format("Unable to set information.  Error: {0}", Marshal.GetLastWin32Error()));
+            
+            Marshal.FreeHGlobal(extendedInfoPtr);
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        private void Dispose(bool disposing)
+        {
+            if (disposed)
+                return;
+
+            if (disposing) { }
+
+            Close();
+            disposed = true;
+        }
+
+        public void Close()
+        {
+            CloseHandle(handle);
+            handle = IntPtr.Zero;
+        }
+
+        public bool AddProcess(IntPtr processHandle)
+        {
+            return AssignProcessToJobObject(handle, processHandle);
+        }
+
+        public bool AddProcess(int processId)
+        {
+            return AddProcess(Process.GetProcessById(processId).Handle);
+        }
+
+    }
+
+    #region Helper classes
+
+    [StructLayout(LayoutKind.Sequential)]
+    struct IO_COUNTERS
+    {
+        public UInt64 ReadOperationCount;
+        public UInt64 WriteOperationCount;
+        public UInt64 OtherOperationCount;
+        public UInt64 ReadTransferCount;
+        public UInt64 WriteTransferCount;
+        public UInt64 OtherTransferCount;
+    }
+
+
+    [StructLayout(LayoutKind.Sequential)]
+    struct JOBOBJECT_BASIC_LIMIT_INFORMATION
+    {
+        public Int64 PerProcessUserTimeLimit;
+        public Int64 PerJobUserTimeLimit;
+        public UInt32 LimitFlags;
+        public UIntPtr MinimumWorkingSetSize;
+        public UIntPtr MaximumWorkingSetSize;
+        public UInt32 ActiveProcessLimit;
+        public UIntPtr Affinity;
+        public UInt32 PriorityClass;
+        public UInt32 SchedulingClass;
+    }
+
+    [StructLayout(LayoutKind.Sequential)]
+    public struct SECURITY_ATTRIBUTES
+    {
+        public UInt32 nLength;
+        public IntPtr lpSecurityDescriptor;
+        public Int32 bInheritHandle;
+    }
+
+    [StructLayout(LayoutKind.Sequential)]
+    struct JOBOBJECT_EXTENDED_LIMIT_INFORMATION
+    {
+        public JOBOBJECT_BASIC_LIMIT_INFORMATION BasicLimitInformation;
+        public IO_COUNTERS IoInfo;
+        public UIntPtr ProcessMemoryLimit;
+        public UIntPtr JobMemoryLimit;
+        public UIntPtr PeakProcessMemoryUsed;
+        public UIntPtr PeakJobMemoryUsed;
+    }
+
+    public enum JobObjectInfoType
+    {
+        AssociateCompletionPortInformation = 7,
+        BasicLimitInformation = 2,
+        BasicUIRestrictions = 4,
+        EndOfJobTimeInformation = 6,
+        ExtendedLimitInformation = 9,
+        SecurityLimitInformation = 5,
+        GroupInformation = 11
+    }
+
+    #endregion
+}

Unity/OpenSeeLauncher.cs

@@ -0,0 +1,283 @@
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Net;
+using System.Text;
+using System.Collections;
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
+using UnityEngine;
+
+namespace OpenSee {
+
+public class OpenSeeLauncher : MonoBehaviour {
+    [Header("Settings")]
+    [Tooltip("If this is left empty, it will default to the OpenSee component on the same game object.")]
+    public OpenSee openSeeTarget;
+    [Tooltip("The numeric index of the camera as in the list returned by the ListCameras function.")]
+    public int cameraIndex = -1;
+    [Tooltip("The path to a video file that should be used as input for the tracker instead of a camera")]
+    public string videoPath = "";
+    [Tooltip("Should facetracker.exe be started automatically with the scene?")]
+    public bool autoStart = false;
+    [Tooltip("If this is enabled, available cameras are automatically printed to the debug console.")]
+    public bool autoListCameras = false;
+    [Tooltip("If this is enabled, no output from the tracker will be printed to the debug console. This cannot be changed while the tracker is running.")]
+    public bool dontPrint = true;
+    [Tooltip("The path to \"facetracker.exe\".")]
+    public string exePath = "facetracker.exe";
+    [Tooltip("The path, where the .onnx model files can be found")]
+    public string modelPath = "models\\";
+    [Tooltip("Additional options that should be passed to the face tracker.")]
+    public string[] commandlineOptions = new string[] { "--silent", "1", "--max-threads", "2", "--model", "1" };
+    [Header("Runtime information")]
+    [Tooltip("This field shows if the tracker is currently alive.")]
+    public bool trackerAlive = false;
+    
+    private string ip;
+    private Process trackerProcess = null;
+    private StringBuilder trackerSB = null;
+    private bool dontPrintNow = false;
+    private Job job = null;
+
+    private bool CheckSetup(bool requireTarget) {
+        if (openSeeTarget == null)
+            openSeeTarget = GetComponent<OpenSee>();
+        if (openSeeTarget == null) {
+            UnityEngine.Debug.LogError("No openSeeTarget is set.");
+            return false;
+        }
+        IPAddress address;
+        if (!File.Exists(exePath)) {
+            UnityEngine.Debug.LogError("Facetracker executable cannot be found.");
+            return false;
+        }
+        if (!Directory.Exists(modelPath)) {
+            UnityEngine.Debug.LogError("Model directory cannot be found.");
+            return false;
+        }
+        if (requireTarget) {
+            ip = openSeeTarget.listenAddress;
+            if (!IPAddress.TryParse(ip, out address)) {
+                UnityEngine.Debug.LogError("No valid IP address was given in the OpenSee component.");
+                return false;
+            }
+            if (openSeeTarget.listenPort < 1 || openSeeTarget.listenPort > 65535) {
+                UnityEngine.Debug.LogError("No valid port was given in the OpenSee component.");
+                return false;
+            }
+            if (cameraIndex >= 0 && videoPath != "") {
+                UnityEngine.Debug.LogError("Entering both a camera index and a video filename is not valid.");
+                return false;
+            }
+            if (cameraIndex < 0 && !File.Exists(videoPath)) {
+                if (videoPath != "") {
+                    UnityEngine.Debug.LogError("The given video file does not exist.");
+                } else {
+                    UnityEngine.Debug.LogError("No camera or video file was given.");
+                }
+                return false;
+            }
+        }
+        if (job == null) {
+            job = new Job();
+        }
+        return true;
+    }
+    
+    public string[] ListCameras() {
+        if (!CheckSetup(false))
+            return null;
+        
+        StringBuilder stringBuilder;
+        ProcessStartInfo processStartInfo;
+        Process process;
+
+        stringBuilder = new StringBuilder();
+
+        processStartInfo = new ProcessStartInfo();
+        processStartInfo.CreateNoWindow = true;
+        processStartInfo.RedirectStandardOutput = true;
+        processStartInfo.RedirectStandardInput = true;
+        processStartInfo.RedirectStandardError = true;
+        processStartInfo.UseShellExecute = false;
+        processStartInfo.FileName = exePath;
+        processStartInfo.Arguments = "--list-cameras 2";
+        
+        process = new Process();
+        process.StartInfo = processStartInfo;
+        process.EnableRaisingEvents = true;
+        process.OutputDataReceived += new DataReceivedEventHandler(delegate(object sender, DataReceivedEventArgs e) {
+            stringBuilder.Append(e.Data);
+            stringBuilder.Append("\n");
+        });
+        process.Start();
+        job.AddProcess(process.Handle);
+        process.BeginOutputReadLine();
+        process.WaitForExit();
+        process.CancelOutputRead();
+        
+        string cameraInfo = stringBuilder.ToString();
+        string[] cameras = cameraInfo.Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.RemoveEmptyEntries);
+        return cameras;
+    }
+    
+    // From: http://csharptest.net/529/how-to-correctly-escape-command-line-arguments-in-c/index.html
+    private static string EscapeArguments(params string[] args)
+    {
+        StringBuilder arguments = new StringBuilder();
+        Regex invalidChar = new Regex("[\x00\x0a\x0d]");//  these can not be escaped
+        Regex needsQuotes = new Regex(@"\s|""");//          contains whitespace or two quote characters
+        Regex escapeQuote = new Regex(@"(\\*)(""|$)");//    one or more '\' followed with a quote or end of string
+        for (int carg = 0; args != null && carg < args.Length; carg++)
+        {
+            if (args[carg] == null) { throw new ArgumentNullException("args[" + carg + "]"); }
+            if (invalidChar.IsMatch(args[carg])) { throw new ArgumentOutOfRangeException("args[" + carg + "]"); }
+            if (args[carg] == String.Empty) { arguments.Append("\"\""); }
+            else if (!needsQuotes.IsMatch(args[carg])) { arguments.Append(args[carg]); }
+            else
+            {
+                arguments.Append('"');
+                arguments.Append(escapeQuote.Replace(args[carg], m => m.Groups[1].Value + m.Groups[1].Value + (m.Groups[2].Value == "\"" ? "\\\"" : "")));
+                arguments.Append('"');
+            }
+            if (carg + 1 < args.Length)
+                arguments.Append(' ');
+        }
+        return arguments.ToString();
+    }
+    
+    public bool StartTracker() {
+        if (!CheckSetup(true))
+            return false;
+
+        ProcessStartInfo processStartInfo;
+        
+        processStartInfo = new ProcessStartInfo();
+        processStartInfo.CreateNoWindow = true;
+        processStartInfo.RedirectStandardOutput = true;
+        processStartInfo.RedirectStandardInput = true;
+        processStartInfo.RedirectStandardError = true;
+        processStartInfo.UseShellExecute = false;
+        processStartInfo.FileName = exePath;
+        
+        List<string> arguments = new List<string>();
+        arguments.Add("--ip");
+        arguments.Add(ip);
+        arguments.Add("--port");
+        arguments.Add(openSeeTarget.listenPort.ToString());
+        arguments.Add("--model-dir");
+        arguments.Add(modelPath);
+        arguments.Add("--capture");
+        if (videoPath != "" && File.Exists(videoPath))
+            arguments.Add(videoPath);
+        else
+            arguments.Add(cameraIndex.ToString());
+        foreach (string argument in commandlineOptions) {
+            arguments.Add(argument);
+        }
+        string argumentString = EscapeArguments(arguments.ToArray());
+        processStartInfo.Arguments = argumentString;
+        
+        if (trackerProcess != null && !trackerProcess.HasExited) {
+            trackerProcess.CloseMainWindow();
+            trackerProcess.Close();
+            if (!dontPrintNow) {
+                try {
+                    trackerProcess.CancelOutputRead();
+                    trackerProcess.CancelErrorRead();
+                } catch {}
+            }
+        }
+
+        trackerSB = new StringBuilder();
+        trackerProcess = new Process();
+        trackerProcess.StartInfo = processStartInfo;
+        dontPrintNow = dontPrint;
+        if (!dontPrintNow) {
+            trackerProcess.EnableRaisingEvents = true;
+            trackerProcess.OutputDataReceived += new DataReceivedEventHandler(delegate(object sender, DataReceivedEventArgs e) {
+                trackerSB.Append(e.Data);
+                trackerSB.Append("\n");
+            });
+            trackerProcess.ErrorDataReceived += new DataReceivedEventHandler(delegate(object sender, DataReceivedEventArgs e) {
+                trackerSB.Append(e.Data);
+                trackerSB.Append("\n");
+            });
+        }
+        trackerProcess.Start();
+        job.AddProcess(trackerProcess.Handle);
+        if (!dontPrintNow) {
+            trackerProcess.BeginOutputReadLine();
+            trackerProcess.BeginErrorReadLine();
+        }
+
+        return !trackerProcess.HasExited;
+    }
+    
+    public void StopTracker() {
+        if (trackerProcess != null && !trackerProcess.HasExited) {
+            trackerProcess.CloseMainWindow();
+            trackerProcess.Close();
+            if (!dontPrintNow) {
+                try {
+                    trackerProcess.CancelOutputRead();
+                    trackerProcess.CancelErrorRead();
+                } catch {}
+            }
+        }
+        trackerProcess = null;
+    }
+    
+    public void Start() {
+        if (autoStart)
+            StartTracker();
+        if (autoListCameras) {
+            string[] cameras = ListCameras();
+            if (cameras == null || cameras.Length < 1) {
+                UnityEngine.Debug.Log("No cameras found.");
+                return;
+            }
+            UnityEngine.Debug.Log("Cameras:");
+            for (int i = 0; i < cameras.Length; i++) {
+                UnityEngine.Debug.Log(i + " = " + cameras[i]);
+            }
+        }
+    }
+    
+    public void Update() {
+        if (trackerProcess != null)
+            trackerAlive = !trackerProcess.HasExited;
+        else {
+            trackerAlive = false;
+            return;
+        }
+        if (dontPrintNow || trackerSB == null)
+            return;
+        int len = trackerSB.Length;
+        if (len > 0) {
+            string output = trackerSB.ToString();
+            UnityEngine.Debug.Log("Tracker: " + output);
+            trackerSB.Clear();
+        }
+    }
+    
+    private void CleanJob() {
+        if (job != null) {
+            job.Dispose();
+            job = null;
+        }
+    }
+    
+    public void OnDestroy() {
+        StopTracker();
+        CleanJob();
+    }
+    
+    public void OnApplicationQuit() {
+        StopTracker();
+        CleanJob();
+    }
+}
+
+}

Unity/OpenSeeShowPoints.cs

@@ -0,0 +1,109 @@
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+
+namespace OpenSee {
+
+public class OpenSeeShowPoints : MonoBehaviour {
+    public OpenSee openSee = null;
+    public bool show3DPoints = true;
+    public bool applyTranslation = false;
+    public bool applyRotation = false;
+    private OpenSee.OpenSeeData[] openSeeData;
+    private GameObject[] gameObjects;
+    private GameObject centerBall;
+    private double updated = 0.0;
+
+	void Start () {
+        if (openSee == null) {
+            openSee = GetComponent<OpenSee>();
+        }
+        gameObjects = new GameObject[66];
+        for (int i = 0; i < 66; i++) {
+            gameObjects[i] = GameObject.CreatePrimitive(PrimitiveType.Sphere);
+            gameObjects[i].name = "Point " + (i + 1);
+            gameObjects[i].transform.SetParent(transform);
+            gameObjects[i].transform.localScale = new Vector3(0.025f, 0.025f, 0.025f);
+        }
+        centerBall = GameObject.CreatePrimitive(PrimitiveType.Sphere);
+        centerBall.name = "Center";
+        centerBall.transform.SetParent(transform);
+        centerBall.transform.localScale = new Vector3(0.1f, 0.1f, 0.1f);
+	}
+    
+	void Update () {
+        if (!openSee)
+            return;
+        openSeeData = openSee.trackingData;
+        if (openSeeData == null || openSeeData.Length < 1)
+            return;
+        if (openSeeData[0].time > updated) {
+            updated = openSeeData[0].time;
+        } else {
+            return;
+        }
+        if (show3DPoints) {
+            centerBall.gameObject.SetActive(false);
+            for (int i = 0; i < 66; i++) {
+                if (openSeeData[0].got3DPoints) {
+                    Renderer renderer = gameObjects[i].GetComponent<Renderer>();
+                    renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData[0].confidence[i]));
+                    gameObjects[i].transform.localPosition = openSeeData[0].points3D[i];
+                } else {
+                    Renderer renderer = gameObjects[i].GetComponent<Renderer>();
+                    renderer.material.SetColor("_Color", Color.cyan);
+                }
+            }
+            if (applyTranslation) {
+                transform.position = openSeeData[0].translation;
+            }
+            if (applyRotation) {
+                transform.eulerAngles = openSeeData[0].rotation;
+            }
+        } else {
+            centerBall.gameObject.SetActive(false);
+            Vector3 center = new Vector3(0.0f, 0.0f, 0.0f);
+            float minX = 10000.0f;
+            float minY = 10000.0f;
+            float maxX = -1.0f;
+            float maxY = -1.0f;
+            for (int i = 0; i < 66; i++) {
+                float x = openSeeData[0].points[i].x;
+                float y = -openSeeData[0].points[i].y;
+                if (minX > x) {
+                    minX = x;
+                }
+                if (minY > y) {
+                    minY = y;
+                }
+                if (maxX < x) {
+                    maxX = x;
+                }
+                if (maxY < y) {
+                    maxY = y;
+                }
+                center += new Vector3(x, y, 0.0f);
+            }
+            center = center / 66;
+            center = center - new Vector3(minX, minY, 0.0f);
+            center.x = center.x / (maxX - minX);
+            center.y = center.y / (maxX - minX);
+            center.z = 0.5f;
+            centerBall.transform.localPosition = center;
+            
+            for (int i = 0; i < 66; i++) {
+                Renderer renderer = gameObjects[i].GetComponent<Renderer>();
+                renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData[0].confidence[i]));
+                float x = openSeeData[0].points[i].x;
+                float y = -openSeeData[0].points[i].y;
+                Vector3 position = new Vector3(x, y, 0.0f);
+                position = position - new Vector3(minX, minY, 0.0f);
+                position.x = position.x / (maxX - minX);
+                position.y = position.y / (maxX - minX);
+                gameObjects[i].transform.localPosition = position;
+            }
+        }
+	}
+}
+
+}