public async Task ProcessReceivedPacket(ReceivedPacketInfo receivedPacket) { Debug.WriteLine("Process Received Packet"); long recvFrameID = receivedPacket.frameID; string recvEngineID = receivedPacket.engineID; SentPacketInfo sentPacket = null; bool isSuccess; // increase appropriate amount of tokens int increaseCount = 0; for (long frameID = _prevRecvFrameID + 1; frameID < recvFrameID; frameID++) { sentPacket = null; if (Const.IS_EXPERIMENT) { // Do not remove since we need to measure latency even for the late response isSuccess = _sentPackets.TryGetValue(frameID, out sentPacket); } else { isSuccess = _sentPackets.TryRemove(frameID, out sentPacket); } if (isSuccess == true) { increaseCount++; } } Debug.WriteLine("Increasing Tokens by " + increaseCount); IncreaseTokens(increaseCount); // deal with the current response isSuccess = _sentPackets.TryGetValue(recvFrameID, out sentPacket); if (isSuccess == true) { // do not increase token if have already received duplicated ack if (recvFrameID > _prevRecvFrameID) { IncreaseTokens(1); } if (Const.IS_EXPERIMENT) { string log = recvFrameID + "\t" + recvEngineID + "\t" + sentPacket.generatedTime + "\t" + sentPacket.compressedTime + "\t" + receivedPacket.msgRecvTime + "\t" + receivedPacket.guidanceDoneTime + "\t" + receivedPacket.status; await _myLogger.WriteString(log + "\n"); } } if (recvFrameID > _prevRecvFrameID) { _prevRecvFrameID = recvFrameID; } }
private void StartResultReceivingThread() { IAsyncAction asyncAction = Windows.System.Threading.ThreadPool.RunAsync( async(workItem) => { while (true) { // receive current time at server string recvMsg = await receiveMsgAsync(_resultReceivingSocketReader); Debug.WriteLine(recvMsg); JsonObject obj = JsonValue.Parse(recvMsg).GetObject(); string status = null; string result = null; long frameID = -1; string engineID = null; try { status = obj.GetNamedString("status"); result = obj.GetNamedString("result"); frameID = (long)obj.GetNamedNumber("frame_id"); engineID = obj.GetNamedString("engine_id"); } catch (Exception) { Debug.WriteLine("the return message has no status field"); return; } ReceivedPacketInfo receivedPacketInfo = new ReceivedPacketInfo(frameID, engineID, status); receivedPacketInfo.setMsgRecvTime(GetTimeMillis()); if (!status.Equals("success")) { receivedPacketInfo.setGuidanceDoneTime(GetTimeMillis()); await _tokenController.ProcessReceivedPacket(receivedPacketInfo); continue; } if (result != null) { /* parsing result */ JsonObject resultJSON = JsonValue.Parse(result).GetObject(); // image guidance string imageFeedbackString = null; try { imageFeedbackString = resultJSON.GetNamedString("image"); } catch (Exception) { Debug.WriteLine("no image guidance found"); } if (imageFeedbackString != null) { IBuffer buffer = CryptographicBuffer.DecodeFromBase64String(imageFeedbackString); //byte[] imageFeedbackBytes; //CryptographicBuffer.CopyToByteArray(buffer, out imageFeedbackBytes); using (var stream = new InMemoryRandomAccessStream()) { using (var dataWriter = new DataWriter(stream)) { dataWriter.WriteBuffer(buffer); await dataWriter.StoreAsync(); BitmapDecoder decoder = await BitmapDecoder.CreateAsync(stream); SoftwareBitmap imageFeedback = await decoder.GetSoftwareBitmapAsync(); SoftwareBitmap imageFeedbackDisplay = SoftwareBitmap.Convert(imageFeedback, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied); await Windows.ApplicationModel.Core.CoreApplication.MainView.CoreWindow.Dispatcher.RunAsync(CoreDispatcherPriority.Normal, async() => { var sbSource = new SoftwareBitmapSource(); await sbSource.SetBitmapAsync(imageFeedbackDisplay); GuidanceImage.Source = sbSource; }); } } } // speech guidance string speechFeedback = null; try { speechFeedback = resultJSON.GetNamedString("speech"); } catch (Exception) { Debug.WriteLine("no speech guidance found"); } if (speechFeedback != null) { SpeechSynthesisStream stream = null; using (SpeechSynthesizer synthesizer = new SpeechSynthesizer()) { stream = await synthesizer.SynthesizeTextToStreamAsync(speechFeedback); } // Send the stream to the media object. await Windows.ApplicationModel.Core.CoreApplication.MainView.CoreWindow.Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () => { _mediaElement.SetSource(stream, stream.ContentType); _mediaElement.Play(); }); } receivedPacketInfo.setGuidanceDoneTime(GetTimeMillis()); await _tokenController.ProcessReceivedPacket(receivedPacketInfo); } } }); // A reference to the work item is cached so that we can trigger a // cancellation when the user presses the Cancel button. _resultReceivingWorkItem = asyncAction; }
private void StartResultReceivingThread() { IAsyncAction asyncAction = Windows.System.Threading.ThreadPool.RunAsync( async(workItem) => { while (true) { // receive current time at server string recvMsg = await receiveMsgAsync(_resultReceivingSocketReader); UnityEngine.Debug.Log(recvMsg); JsonObject obj = JsonValue.Parse(recvMsg).GetObject(); string status = null; string result = null; long frameID = -1; string engineID = null; if (obj.ContainsKey("status")) { status = obj.GetNamedString("status"); result = obj.GetNamedString("result"); frameID = (long)obj.GetNamedNumber("frame_id"); engineID = obj.GetNamedString("engine_id"); } else { UnityEngine.Debug.Log("the return message has no status field"); return; } ReceivedPacketInfo receivedPacketInfo = new ReceivedPacketInfo(frameID, engineID, status); receivedPacketInfo.setMsgRecvTime(GetTimeMillis()); if (!status.Equals("success")) { receivedPacketInfo.setGuidanceDoneTime(GetTimeMillis()); _tokenController.ProcessReceivedPacket(receivedPacketInfo); continue; } if (result != null) { /* parsing result */ JsonObject resultJSON = JsonValue.Parse(result).GetObject(); // speech guidance if (resultJSON.ContainsKey("speech")) { _speechFeedback = resultJSON.GetNamedString("speech"); //UnityEngine.Debug.Log("speech guidance found"); textToSpeechManager.SpeakText(_speechFeedback); } // hologram guidance if (resultJSON.ContainsKey("holo_object")) { _guidanceObject = resultJSON.GetNamedString("holo_object"); if (!_guidanceObject.Equals("none")) { JsonObject holoPosInfo = resultJSON.GetNamedObject("holo_location"); UnityEngine.Debug.Log("Hologram guidance: " + holoPosInfo); float x = (float)holoPosInfo.GetNamedNumber("x"); float y = (float)holoPosInfo.GetNamedNumber("y"); float depth = (float)holoPosInfo.GetNamedNumber("depth"); SentPacketInfo p = _tokenController.GetSentPacket(frameID); _guidancePos = Pixel2WorldPos(x, y, depth, p.projectionMatrix, p.cameraToWorldMatrix); } _guidancePosReady = true; } receivedPacketInfo.setGuidanceDoneTime(GetTimeMillis()); _tokenController.ProcessReceivedPacket(receivedPacketInfo); } } }); // A reference to the work item is cached so that we can trigger a // cancellation when the user presses the Cancel button. _resultReceivingWorkItem = asyncAction; }