public async Task ProcessReceivedPacket(ReceivedPacketInfo receivedPacket)
{
Debug.WriteLine("Process Received Packet");
long recvFrameID = receivedPacket.frameID;
string recvEngineID = receivedPacket.engineID;
SentPacketInfo sentPacket = null;
bool isSuccess;
// increase appropriate amount of tokens
int increaseCount = 0;
for (long frameID = _prevRecvFrameID + 1; frameID < recvFrameID; frameID++)
{
sentPacket = null;
if (Const.IS_EXPERIMENT)
{
// Do not remove since we need to measure latency even for the late response
isSuccess = _sentPackets.TryGetValue(frameID, out sentPacket);
}
else
{
isSuccess = _sentPackets.TryRemove(frameID, out sentPacket);
}
if (isSuccess == true)
{
increaseCount++;
}
}
Debug.WriteLine("Increasing Tokens by " + increaseCount);
IncreaseTokens(increaseCount);
// deal with the current response
isSuccess = _sentPackets.TryGetValue(recvFrameID, out sentPacket);
if (isSuccess == true)
{
// do not increase token if have already received duplicated ack
if (recvFrameID > _prevRecvFrameID)
{
IncreaseTokens(1);
}
if (Const.IS_EXPERIMENT)
{
string log = recvFrameID + "\t" + recvEngineID + "\t" +
sentPacket.generatedTime + "\t" + sentPacket.compressedTime + "\t" +
receivedPacket.msgRecvTime + "\t" + receivedPacket.guidanceDoneTime + "\t" +
receivedPacket.status;
await _myLogger.WriteString(log + "\n");
}
}
if (recvFrameID > _prevRecvFrameID)
{
_prevRecvFrameID = recvFrameID;
}
}
private void StartResultReceivingThread()
{
IAsyncAction asyncAction = Windows.System.Threading.ThreadPool.RunAsync(
async(workItem) =>
{
while (true)
{
// receive current time at server
string recvMsg = await receiveMsgAsync(_resultReceivingSocketReader);
Debug.WriteLine(recvMsg);
JsonObject obj = JsonValue.Parse(recvMsg).GetObject();
string status = null;
string result = null;
long frameID = -1;
string engineID = null;
try
{
status = obj.GetNamedString("status");
result = obj.GetNamedString("result");
frameID = (long)obj.GetNamedNumber("frame_id");
engineID = obj.GetNamedString("engine_id");
}
catch (Exception)
{
Debug.WriteLine("the return message has no status field");
return;
}
ReceivedPacketInfo receivedPacketInfo = new ReceivedPacketInfo(frameID, engineID, status);
receivedPacketInfo.setMsgRecvTime(GetTimeMillis());
if (!status.Equals("success"))
{
receivedPacketInfo.setGuidanceDoneTime(GetTimeMillis());
await _tokenController.ProcessReceivedPacket(receivedPacketInfo);
continue;
}
if (result != null)
{
/* parsing result */
JsonObject resultJSON = JsonValue.Parse(result).GetObject();
// image guidance
string imageFeedbackString = null;
try
{
imageFeedbackString = resultJSON.GetNamedString("image");
}
catch (Exception)
{
Debug.WriteLine("no image guidance found");
}
if (imageFeedbackString != null)
{
IBuffer buffer = CryptographicBuffer.DecodeFromBase64String(imageFeedbackString);
//byte[] imageFeedbackBytes;
//CryptographicBuffer.CopyToByteArray(buffer, out imageFeedbackBytes);
using (var stream = new InMemoryRandomAccessStream())
{
using (var dataWriter = new DataWriter(stream))
{
dataWriter.WriteBuffer(buffer);
await dataWriter.StoreAsync();
BitmapDecoder decoder = await BitmapDecoder.CreateAsync(stream);
SoftwareBitmap imageFeedback = await decoder.GetSoftwareBitmapAsync();
SoftwareBitmap imageFeedbackDisplay = SoftwareBitmap.Convert(imageFeedback, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied);
await Windows.ApplicationModel.Core.CoreApplication.MainView.CoreWindow.Dispatcher.RunAsync(CoreDispatcherPriority.Normal, async() => {
var sbSource = new SoftwareBitmapSource();
await sbSource.SetBitmapAsync(imageFeedbackDisplay);
GuidanceImage.Source = sbSource;
});
}
}
}
// speech guidance
string speechFeedback = null;
try
{
speechFeedback = resultJSON.GetNamedString("speech");
}
catch (Exception)
{
Debug.WriteLine("no speech guidance found");
}
if (speechFeedback != null)
{
SpeechSynthesisStream stream = null;
using (SpeechSynthesizer synthesizer = new SpeechSynthesizer())
{
stream = await synthesizer.SynthesizeTextToStreamAsync(speechFeedback);
}
// Send the stream to the media object.
await Windows.ApplicationModel.Core.CoreApplication.MainView.CoreWindow.Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () => {
_mediaElement.SetSource(stream, stream.ContentType);
_mediaElement.Play();
});
}
receivedPacketInfo.setGuidanceDoneTime(GetTimeMillis());
await _tokenController.ProcessReceivedPacket(receivedPacketInfo);
}
}
});
// A reference to the work item is cached so that we can trigger a
// cancellation when the user presses the Cancel button.
_resultReceivingWorkItem = asyncAction;
}
private void StartResultReceivingThread()
{
IAsyncAction asyncAction = Windows.System.Threading.ThreadPool.RunAsync(
async(workItem) =>
{
while (true)
{
// receive current time at server
string recvMsg = await receiveMsgAsync(_resultReceivingSocketReader);
UnityEngine.Debug.Log(recvMsg);
JsonObject obj = JsonValue.Parse(recvMsg).GetObject();
string status = null;
string result = null;
long frameID = -1;
string engineID = null;
if (obj.ContainsKey("status"))
{
status = obj.GetNamedString("status");
result = obj.GetNamedString("result");
frameID = (long)obj.GetNamedNumber("frame_id");
engineID = obj.GetNamedString("engine_id");
}
else
{
UnityEngine.Debug.Log("the return message has no status field");
return;
}
ReceivedPacketInfo receivedPacketInfo = new ReceivedPacketInfo(frameID, engineID, status);
receivedPacketInfo.setMsgRecvTime(GetTimeMillis());
if (!status.Equals("success"))
{
receivedPacketInfo.setGuidanceDoneTime(GetTimeMillis());
_tokenController.ProcessReceivedPacket(receivedPacketInfo);
continue;
}
if (result != null)
{
/* parsing result */
JsonObject resultJSON = JsonValue.Parse(result).GetObject();
// speech guidance
if (resultJSON.ContainsKey("speech"))
{
_speechFeedback = resultJSON.GetNamedString("speech");
//UnityEngine.Debug.Log("speech guidance found");
textToSpeechManager.SpeakText(_speechFeedback);
}
// hologram guidance
if (resultJSON.ContainsKey("holo_object"))
{
_guidanceObject = resultJSON.GetNamedString("holo_object");
if (!_guidanceObject.Equals("none"))
{
JsonObject holoPosInfo = resultJSON.GetNamedObject("holo_location");
UnityEngine.Debug.Log("Hologram guidance: " + holoPosInfo);
float x = (float)holoPosInfo.GetNamedNumber("x");
float y = (float)holoPosInfo.GetNamedNumber("y");
float depth = (float)holoPosInfo.GetNamedNumber("depth");
SentPacketInfo p = _tokenController.GetSentPacket(frameID);
_guidancePos = Pixel2WorldPos(x, y, depth, p.projectionMatrix, p.cameraToWorldMatrix);
}
_guidancePosReady = true;
}
receivedPacketInfo.setGuidanceDoneTime(GetTimeMillis());
_tokenController.ProcessReceivedPacket(receivedPacketInfo);
}
}
});
// A reference to the work item is cached so that we can trigger a
// cancellation when the user presses the Cancel button.
_resultReceivingWorkItem = asyncAction;
}
