// checks if VisualGestureManager is initialized or not
private IEnumerator CheckVisualGestureManager()
{
// wait for 2 seconds
yield return(new WaitForSeconds(2f));
string sStatusMsg = string.Empty;
VisualGestureManager vgbManager = VisualGestureManager.Instance;
if (!vgbManager)
{
sStatusMsg = "VisualGestureManager is missing!";
}
else if (!vgbManager.IsVisualGestureInitialized())
{
sStatusMsg = "VisualGestureManager not initialized! Check the log-file for details.";
}
else
{
LogToConsole("VisualGestureManager is ready.");
}
if (sStatusMsg.Length > 0)
{
LogErrorToConsole(sStatusMsg);
}
}
// enables or disables the visual gestures component
public void EnableVisualGestures(bool bEnable)
{
VisualGestureManager vgbManager = gameObject.GetComponent <VisualGestureManager>();
if (vgbManager)
{
vgbManager.enabled = bEnable;
LogToConsole("Visual gestures are " + (bEnable ? "enabled" : "disabled"));
if (bEnable)
{
StartCoroutine(CheckVisualGestureManager());
}
// enable SimpleVisualGestureListener as well
SimpleVisualGestureListener vgbListener = gameObject.GetComponent <SimpleVisualGestureListener>();
if (vgbListener)
{
vgbListener.enabled = bEnable;
}
}
else
{
LogErrorToConsole("VisualGestureManager-component not found.");
}
}
void OnDestroy()
{
if (isVisualGestureInitialized)
{
// finish visual gesture tracking
FinishVisualGestures();
}
isVisualGestureInitialized = false;
instance = null;
}
//----------------------------------- end of public functions --------------------------------------//
void Awake()
{
instance = this;
}
//----------------------------------- end of public functions --------------------------------------//
void Start()
{
try
{
// get sensor data
KinectManager kinectManager = KinectManager.Instance;
KinectInterop.SensorData sensorData = kinectManager != null?kinectManager.GetSensorData() : null;
if (sensorData == null || sensorData.sensorInterface == null)
{
throw new Exception("Visual gesture tracking cannot be started, because the KinectManager is missing or not initialized.");
}
if (sensorData.sensorInterface.GetSensorPlatform() != KinectInterop.DepthSensorPlatform.KinectSDKv2)
{
throw new Exception("Visual gesture tracking is only supported by Kinect SDK v2");
}
// ensure the needed dlls are in place and face tracking is available for this interface
bool bNeedRestart = false;
if (IsVisualGesturesAvailable(ref bNeedRestart))
{
if (bNeedRestart)
{
KinectInterop.RestartLevel(gameObject, "VG");
return;
}
}
else
{
throw new Exception("Visual gesture tracking is not supported!");
}
// initialize visual gesture tracker
if (!InitVisualGestures())
{
throw new Exception("Visual gesture tracking could not be initialized.");
}
// try to automatically detect the available gesture listeners in the scene
if (visualGestureListeners.Count == 0)
{
MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[];
foreach (MonoBehaviour monoScript in monoScripts)
{
if (typeof(VisualGestureListenerInterface).IsAssignableFrom(monoScript.GetType()) &&
monoScript.enabled)
{
visualGestureListeners.Add(monoScript);
}
}
}
// all set
instance = this;
isVisualGestureInitialized = true;
}
catch (DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = "Please check the Kinect and FT-Library installations.";
}
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = ex.Message;
}
}
}
void Start()
{
// get references to the needed components
manager = KinectManager.Instance;
gestureManager = VisualGestureManager.Instance;
speechManager = SpeechManager.Instance;
// create lz4 compressor & decompressor
decompressor = LZ4Sharp.LZ4DecompressorFactory.CreateNew();
compressor = LZ4Sharp.LZ4CompressorFactory.CreateNew();
try
{
if (serverHost != string.Empty && serverHost != "0.0.0.0" && serverPort != 0)
{
byte error = 0;
clientConnId = NetworkTransport.Connect(clientHostId, serverHost, serverPort, 0, out error);
if (error == (byte)NetworkError.Ok)
{
Debug.Log("Connecting to the server - " + serverHost + ":" + serverPort);
if (statusText)
{
statusText.text = "Connecting to the server...";
}
}
else
{
throw new UnityException("Error while connecting: " + (NetworkError)error);
}
}
else if (broadcastPort > 0)
{
Debug.Log("Waiting for the server...");
if (statusText)
{
statusText.text = "Waiting for the server...";
}
}
else
{
Debug.Log("Server settings are not set. Cannot connect.");
if (statusText)
{
statusText.text = "Server settings are not set. Cannot connect.";
}
}
}
catch (System.Exception ex)
{
Debug.LogError(ex.Message + "\n" + ex.StackTrace);
if (statusText)
{
statusText.text = ex.Message;
}
}
}
void Update()
{
int recHostId;
int connectionId;
int recChannelId;
int dataSize;
bool connListUpdated = false;
if (backgroundImage && backgroundImage.texture == null)
{
backgroundImage.texture = manager ? manager.GetUsersLblTex() : null;
}
if (faceManager == null)
{
faceManager = FacetrackingManager.Instance;
}
if (gestureManager == null)
{
gestureManager = VisualGestureManager.Instance;
}
if (speechManager == null)
{
speechManager = SpeechManager.Instance;
}
try
{
byte error = 0;
NetworkEventType recData = NetworkTransport.Receive(out recHostId, out connectionId, out recChannelId, recBuffer, bufferSize, out dataSize, out error);
switch (recData)
{
case NetworkEventType.Nothing: //1
break;
case NetworkEventType.ConnectEvent: //2
if (recHostId == serverHostId && recChannelId == serverChannelId &&
!dictConnection.ContainsKey(connectionId))
{
HostConnection conn = new HostConnection();
conn.hostId = recHostId;
conn.connectionId = connectionId;
conn.channelId = recChannelId;
conn.keepAlive = true;
conn.reqDataType = "ka,kb,km,kh";
//conn.matrixSent = false;
dictConnection[connectionId] = conn;
connListUpdated = true;
//LogToConsole(connectionId + "-conn: " + conn.reqDataType);
}
// // reset chunked face messages
// sendFvMsg = string.Empty;
// sendFvNextOfs = 0;
//
// sendFtMsg = string.Empty;
// sendFtNextOfs = 0;
break;
case NetworkEventType.DataEvent: //3
if (recHostId == serverHostId && recChannelId == serverChannelId &&
dictConnection.ContainsKey(connectionId))
{
HostConnection conn = dictConnection[connectionId];
int decompSize = 0;
if (decompressor != null && (recBuffer[0] > 127 || recBuffer[0] < 32))
{
decompSize = decompressor.Decompress(recBuffer, 0, compressBuffer, 0, dataSize);
}
else
{
System.Buffer.BlockCopy(recBuffer, 0, compressBuffer, 0, dataSize);
decompSize = dataSize;
}
string sRecvMessage = System.Text.Encoding.UTF8.GetString(compressBuffer, 0, decompSize);
if (sRecvMessage.StartsWith("ka"))
{
if (sRecvMessage == "ka") // vr-examples v1.0 keep-alive message
{
sRecvMessage = "ka,kb,km,kh";
}
conn.keepAlive = true;
conn.reqDataType = sRecvMessage;
dictConnection[connectionId] = conn;
//LogToConsole(connectionId + "-recv: " + conn.reqDataType);
// check for SR phrase-reset
int iIndexSR = sRecvMessage.IndexOf(",sr");
if (iIndexSR >= 0 && speechManager)
{
speechManager.ClearPhraseRecognized();
//LogToConsole("phrase cleared");
}
}
}
break;
case NetworkEventType.DisconnectEvent: //4
if (dictConnection.ContainsKey(connectionId))
{
dictConnection.Remove(connectionId);
connListUpdated = true;
}
break;
}
if (connListUpdated)
{
// get all connection IDs
alConnectionId.Clear();
alConnectionId.AddRange(dictConnection.Keys);
// display the number of connections
StringBuilder sbConnStatus = new StringBuilder();
sbConnStatus.AppendFormat("Server running: {0} connection(s)", dictConnection.Count);
foreach (int connId in dictConnection.Keys)
{
HostConnection conn = dictConnection[connId];
int iPort = 0; string sAddress = string.Empty; NetworkID network; NodeID destNode;
NetworkTransport.GetConnectionInfo(conn.hostId, conn.connectionId, out sAddress, out iPort, out network, out destNode, out error);
if (error == (int)NetworkError.Ok)
{
sbConnStatus.AppendLine().Append(" ").Append(sAddress).Append(":").Append(iPort);
}
}
LogToConsole(sbConnStatus.ToString());
if (connStatusText)
{
connStatusText.text = sbConnStatus.ToString();
}
}
// send body frame to available connections
const char delimiter = ',';
string sBodyFrame = manager ? manager.GetBodyFrameData(ref liRelTime, ref fCurrentTime, delimiter) : string.Empty;
if (sBodyFrame.Length > 0 && dictConnection.Count > 0)
{
StringBuilder sbSendMessage = new StringBuilder();
bool bFaceParamsRequested = IsFaceParamsRequested();
sbSendMessage.Append(manager.GetWorldMatrixData(delimiter)).Append('|');
sbSendMessage.Append(sBodyFrame).Append('|');
sbSendMessage.Append(manager.GetBodyHandData(ref liRelTime, delimiter)).Append('|');
if (bFaceParamsRequested && faceManager && faceManager.IsFaceTrackingInitialized())
{
sbSendMessage.Append(faceManager.GetFaceParamsAsCsv(delimiter)).Append('|');
}
if (gestureManager && gestureManager.IsVisualGestureInitialized())
{
sbSendMessage.Append(gestureManager.GetGestureDataAsCsv(delimiter)).Append('|');
}
if (speechManager && speechManager.IsSapiInitialized())
{
sbSendMessage.Append(speechManager.GetSpeechDataAsCsv(delimiter)).Append('|');
}
if (sbSendMessage.Length > 0 && sbSendMessage[sbSendMessage.Length - 1] == '|')
{
sbSendMessage.Remove(sbSendMessage.Length - 1, 1);
}
byte[] btSendMessage = System.Text.Encoding.UTF8.GetBytes(sbSendMessage.ToString());
//Debug.Log("Message " + sbSendMessage.Length + " chars: " + sbSendMessage.ToString());
//Debug.Log("Encoded into " + btSendMessage.Length + " bytes: " + ByteArrayToString(btSendMessage, btSendMessage.Length));
int compSize = 0;
if (compressor != null && btSendMessage.Length > 100 && !websocketHost)
{
compSize = compressor.Compress(btSendMessage, 0, btSendMessage.Length, compressBuffer, 0);
}
else
{
System.Buffer.BlockCopy(btSendMessage, 0, compressBuffer, 0, btSendMessage.Length);
compSize = btSendMessage.Length;
}
//Debug.Log("Compressed into " + compSize + " bytes: " + ByteArrayToString(compressBuffer, compSize));
// // check face-tracking requests
// bool bFaceParams = false, bFaceVertices = false, bFaceUvs = false, bFaceTriangles = false;
// if(faceManager && faceManager.IsFaceTrackingInitialized())
// CheckFacetrackRequests(out bFaceParams, out bFaceVertices, out bFaceUvs, out bFaceTriangles);
//
// byte[] btFaceParams = null;
// if(bFaceParams)
// {
// string sFaceParams = faceManager.GetFaceParamsAsCsv();
// if(!string.IsNullOrEmpty(sFaceParams))
// btFaceParams = System.Text.Encoding.UTF8.GetBytes(sFaceParams);
// }
//
// // next chunk of data for face vertices
// byte[] btFaceVertices = null;
// string sFvMsgHead = string.Empty;
// GetNextFaceVertsChunk(bFaceVertices, bFaceUvs, ref btFaceVertices, out sFvMsgHead);
//
// // next chunk of data for face triangles
// byte[] btFaceTriangles = null;
// string sFtMsgHead = string.Empty;
// GetNextFaceTrisChunk(bFaceTriangles, ref btFaceTriangles, out sFtMsgHead);
foreach (int connId in alConnectionId)
{
HostConnection conn = dictConnection[connId];
if (conn.keepAlive)
{
conn.keepAlive = false;
dictConnection[connId] = conn;
if (conn.reqDataType != null && conn.reqDataType.Contains("kb,"))
{
//LogToConsole(conn.connectionId + "-sendkb: " + conn.reqDataType);
error = 0;
//if(!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, btSendMessage, btSendMessage.Length, out error))
if (!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, compressBuffer, compSize, out error))
{
string sMessage = "Error sending body data via conn " + conn.connectionId + ": " + (NetworkError)error;
LogErrorToConsole(sMessage);
if (serverStatusText)
{
serverStatusText.text = sMessage;
}
}
}
// if(bFaceParams && btFaceParams != null &&
// conn.reqDataType != null && conn.reqDataType.Contains("fp,"))
// {
// //Debug.Log(conn.connectionId + "-sendfp: " + conn.reqDataType);
//
// error = 0;
// if(!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, btFaceParams, btFaceParams.Length, out error))
// {
// string sMessage = "Error sending face params via conn " + conn.connectionId + ": " + (NetworkError)error;
// Debug.LogError(sMessage);
//
// if(serverStatusText)
// {
// serverStatusText.text = sMessage;
// }
// }
// }
//
// if(bFaceVertices && btFaceVertices != null &&
// conn.reqDataType != null && conn.reqDataType.Contains("fv,"))
// {
// //Debug.Log(conn.connectionId + "-sendfv: " + conn.reqDataType + " - " + sFvMsgHead);
//
// error = 0;
// if(!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, btFaceVertices, btFaceVertices.Length, out error))
// {
// string sMessage = "Error sending face verts via conn " + conn.connectionId + ": " + (NetworkError)error;
// Debug.LogError(sMessage);
//
// if(serverStatusText)
// {
// serverStatusText.text = sMessage;
// }
// }
// }
//
// if(bFaceTriangles && btFaceTriangles != null &&
// conn.reqDataType != null && conn.reqDataType.Contains("ft,"))
// {
// //Debug.Log(conn.connectionId + "-sendft: " + conn.reqDataType + " - " + sFtMsgHead);
//
// error = 0;
// if(!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, btFaceTriangles, btFaceTriangles.Length, out error))
// {
// string sMessage = "Error sending face tris via conn " + conn.connectionId + ": " + (NetworkError)error;
// Debug.LogError(sMessage);
//
// if(serverStatusText)
// {
// serverStatusText.text = sMessage;
// }
// }
// }
}
}
}
}
catch (System.Exception ex)
{
LogErrorToConsole(ex.Message + "\n" + ex.StackTrace);
if (serverStatusText)
{
serverStatusText.text = ex.Message;
}
}
}
void OnDestroy()
{
if(isVisualGestureInitialized)
{
// finish visual gesture tracking
FinishVisualGestures();
}
isVisualGestureInitialized = false;
instance = null;
}
//----------------------------------- end of public functions --------------------------------------//
void Start()
{
try
{
// get sensor data
KinectManager kinectManager = KinectManager.Instance;
KinectInterop.SensorData sensorData = kinectManager != null ? kinectManager.GetSensorData() : null;
if(sensorData == null || sensorData.sensorInterface == null)
{
throw new Exception("Visual gesture tracking cannot be started, because the KinectManager is missing or not initialized.");
}
if(sensorData.sensorInterface.GetSensorPlatform() != KinectInterop.DepthSensorPlatform.KinectSDKv2)
{
throw new Exception("Visual gesture tracking is only supported by Kinect SDK v2");
}
// ensure the needed dlls are in place and face tracking is available for this interface
bool bNeedRestart = false;
if(IsVisualGesturesAvailable(ref bNeedRestart))
{
if(bNeedRestart)
{
KinectInterop.RestartLevel(gameObject, "VG");
return;
}
}
else
{
throw new Exception("Visual gesture tracking is not supported!");
}
// initialize visual gesture tracker
if (!InitVisualGestures())
{
throw new Exception("Visual gesture tracking could not be initialized.");
}
// try to automatically detect the available gesture listeners in the scene
if(visualGestureListeners.Count == 0)
{
MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[];
foreach(MonoBehaviour monoScript in monoScripts)
{
if(typeof(VisualGestureListenerInterface).IsAssignableFrom(monoScript.GetType()) &&
monoScript.enabled)
{
visualGestureListeners.Add(monoScript);
}
}
}
// all set
instance = this;
isVisualGestureInitialized = true;
}
catch(DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if(debugText != null)
debugText.GetComponent<GUIText>().text = "Please check the Kinect and FT-Library installations.";
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if(debugText != null)
debugText.GetComponent<GUIText>().text = ex.Message;
}
}
