private void StartResultReceivingThread()
{
IAsyncAction asyncAction = Windows.System.Threading.ThreadPool.RunAsync(
async(workItem) =>
{
while (true)
{
// receive current time at server
string recvMsg = await receiveMsgAsync(_resultReceivingSocketReader);
Debug.WriteLine(recvMsg);
JsonObject obj = JsonValue.Parse(recvMsg).GetObject();
string status = null;
string result = null;
long frameID = -1;
string engineID = null;
try
{
status = obj.GetNamedString("status");
result = obj.GetNamedString("result");
frameID = (long)obj.GetNamedNumber("frame_id");
engineID = obj.GetNamedString("engine_id");
}
catch (Exception)
{
Debug.WriteLine("the return message has no status field");
return;
}
ReceivedPacketInfo receivedPacketInfo = new ReceivedPacketInfo(frameID, engineID, status);
receivedPacketInfo.setMsgRecvTime(GetTimeMillis());
if (!status.Equals("success"))
{
receivedPacketInfo.setGuidanceDoneTime(GetTimeMillis());
await _tokenController.ProcessReceivedPacket(receivedPacketInfo);
continue;
}
if (result != null)
{
/* parsing result */
JsonObject resultJSON = JsonValue.Parse(result).GetObject();
// image guidance
string imageFeedbackString = null;
try
{
imageFeedbackString = resultJSON.GetNamedString("image");
}
catch (Exception)
{
Debug.WriteLine("no image guidance found");
}
if (imageFeedbackString != null)
{
IBuffer buffer = CryptographicBuffer.DecodeFromBase64String(imageFeedbackString);
//byte[] imageFeedbackBytes;
//CryptographicBuffer.CopyToByteArray(buffer, out imageFeedbackBytes);
using (var stream = new InMemoryRandomAccessStream())
{
using (var dataWriter = new DataWriter(stream))
{
dataWriter.WriteBuffer(buffer);
await dataWriter.StoreAsync();
BitmapDecoder decoder = await BitmapDecoder.CreateAsync(stream);
SoftwareBitmap imageFeedback = await decoder.GetSoftwareBitmapAsync();
SoftwareBitmap imageFeedbackDisplay = SoftwareBitmap.Convert(imageFeedback, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied);
await Windows.ApplicationModel.Core.CoreApplication.MainView.CoreWindow.Dispatcher.RunAsync(CoreDispatcherPriority.Normal, async() => {
var sbSource = new SoftwareBitmapSource();
await sbSource.SetBitmapAsync(imageFeedbackDisplay);
GuidanceImage.Source = sbSource;
});
}
}
}
// speech guidance
string speechFeedback = null;
try
{
speechFeedback = resultJSON.GetNamedString("speech");
}
catch (Exception)
{
Debug.WriteLine("no speech guidance found");
}
if (speechFeedback != null)
{
SpeechSynthesisStream stream = null;
using (SpeechSynthesizer synthesizer = new SpeechSynthesizer())
{
stream = await synthesizer.SynthesizeTextToStreamAsync(speechFeedback);
}
// Send the stream to the media object.
await Windows.ApplicationModel.Core.CoreApplication.MainView.CoreWindow.Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () => {
_mediaElement.SetSource(stream, stream.ContentType);
_mediaElement.Play();
});
}
receivedPacketInfo.setGuidanceDoneTime(GetTimeMillis());
await _tokenController.ProcessReceivedPacket(receivedPacketInfo);
}
}
});
// A reference to the work item is cached so that we can trigger a
// cancellation when the user presses the Cancel button.
_resultReceivingWorkItem = asyncAction;
}
private void StartResultReceivingThread()
{
IAsyncAction asyncAction = Windows.System.Threading.ThreadPool.RunAsync(
async(workItem) =>
{
while (true)
{
// receive current time at server
string recvMsg = await receiveMsgAsync(_resultReceivingSocketReader);
UnityEngine.Debug.Log(recvMsg);
JsonObject obj = JsonValue.Parse(recvMsg).GetObject();
string status = null;
string result = null;
long frameID = -1;
string engineID = null;
if (obj.ContainsKey("status"))
{
status = obj.GetNamedString("status");
result = obj.GetNamedString("result");
frameID = (long)obj.GetNamedNumber("frame_id");
engineID = obj.GetNamedString("engine_id");
}
else
{
UnityEngine.Debug.Log("the return message has no status field");
return;
}
ReceivedPacketInfo receivedPacketInfo = new ReceivedPacketInfo(frameID, engineID, status);
receivedPacketInfo.setMsgRecvTime(GetTimeMillis());
if (!status.Equals("success"))
{
receivedPacketInfo.setGuidanceDoneTime(GetTimeMillis());
_tokenController.ProcessReceivedPacket(receivedPacketInfo);
continue;
}
if (result != null)
{
/* parsing result */
JsonObject resultJSON = JsonValue.Parse(result).GetObject();
// speech guidance
if (resultJSON.ContainsKey("speech"))
{
_speechFeedback = resultJSON.GetNamedString("speech");
//UnityEngine.Debug.Log("speech guidance found");
textToSpeechManager.SpeakText(_speechFeedback);
}
// hologram guidance
if (resultJSON.ContainsKey("holo_object"))
{
_guidanceObject = resultJSON.GetNamedString("holo_object");
if (!_guidanceObject.Equals("none"))
{
JsonObject holoPosInfo = resultJSON.GetNamedObject("holo_location");
UnityEngine.Debug.Log("Hologram guidance: " + holoPosInfo);
float x = (float)holoPosInfo.GetNamedNumber("x");
float y = (float)holoPosInfo.GetNamedNumber("y");
float depth = (float)holoPosInfo.GetNamedNumber("depth");
SentPacketInfo p = _tokenController.GetSentPacket(frameID);
_guidancePos = Pixel2WorldPos(x, y, depth, p.projectionMatrix, p.cameraToWorldMatrix);
}
_guidancePosReady = true;
}
receivedPacketInfo.setGuidanceDoneTime(GetTimeMillis());
_tokenController.ProcessReceivedPacket(receivedPacketInfo);
}
}
});
// A reference to the work item is cached so that we can trigger a
// cancellation when the user presses the Cancel button.
_resultReceivingWorkItem = asyncAction;
}