private async Task EvaluateFrame(Windows.Media.VideoFrame videoFrame) { try { var result = await _dnnModel.EvaluateVideoFrameAsync(videoFrame); if (result.DominantResultProbability > 0) { // Further process and surface results to UI on the UI thread UnityEngine.WSA.Application.InvokeOnAppThread(() => { // Measure distance between user's head and gaze ray hit point => distance to object var distMessage = string.Empty; if (_user.GazeHitDistance < 1) { distMessage = string.Format("{0:f0} {1}", _user.GazeHitDistance * 100, "centimeter"); } else { distMessage = string.Format("{0:f1} {1}", _user.GazeHitDistance, "meter"); } // Prepare strings for text and update labels var deviceKind = ShouldUseGpu ? "GPU" : "CPU"; var labelText = $"Predominant objects detected in {result.ElapsedMilliseconds,3:f0}ms on {deviceKind}\n {result.TopResultsFormatted}"; var speechText = string.Format("This {0} a {1} {2} in front of you", result.DominantResultProbability > ProbabilityThreshold ? "is likely" : "might be", result.DominantResultLabel, distMessage); StatusBlock.text = labelText; // Check if the previous result was the same and only progress further if not to avoid a loop of same audio if (!_tts.IsSpeaking() && result.DominantResultLabel != _previousDominantResult) { _tts.StartSpeaking(speechText); _previousDominantResult = result.DominantResultLabel; } }, false); } } catch (Exception ex) { //IsRunning = false; UnityEngine.WSA.Application.InvokeOnAppThread(() => { //StatusBlock.text = $"Error loop: {ex.Message}"; //Debug.LogError(ex); //Debug.LogError(videoFrame.Direct3DSurface == null ? "D3D null" : "D3D set"); //if (videoFrame.Direct3DSurface != null) //{ // Debug.LogError(videoFrame.Direct3DSurface.Description.Format); // Debug.LogError(videoFrame.Direct3DSurface.Description.Width); // Debug.LogError(videoFrame.Direct3DSurface.Description.Height); //} }, false); } }
private async Task EvaluateFrame(Windows.Media.VideoFrame videoFrame) { try { var result = await _dnnModel.PredictImageAsync(videoFrame); var first = result.FirstOrDefault(); if (first != null && first.Probability > .5) { // Further process and surface results to UI on the UI thread UnityEngine.WSA.Application.InvokeOnAppThread(() => { // Measure distance between user's head and gaze ray hit point => distance to object var distMessage = string.Empty; if (_user.GazeHitDistance < 1) { distMessage = string.Format("{0:f0} {1}", _user.GazeHitDistance * 100, "centimeter"); } else { distMessage = string.Format("{0:f1} {1}", _user.GazeHitDistance, "meter"); } // Prepare strings for text and update labels var labelText = $"Found: {first.TagName}"; StatusBlock.text = labelText; }, false); } } catch (Exception ex) { //IsRunning = false; UnityEngine.WSA.Application.InvokeOnAppThread(() => { //StatusBlock.text = $"Error loop: {ex.Message}"; //Debug.LogError(ex); //Debug.LogError(videoFrame.Direct3DSurface == null ? "D3D null" : "D3D set"); //if (videoFrame.Direct3DSurface != null) //{ // Debug.LogError(videoFrame.Direct3DSurface.Description.Format); // Debug.LogError(videoFrame.Direct3DSurface.Description.Width); // Debug.LogError(videoFrame.Direct3DSurface.Description.Height); //} }, false); } }
private async Task EvaluateFrame(Windows.Media.VideoFrame videoFrame) { try { // Get the current network prediction from model and input frame var result = await _networkModel.EvaluateVideoFrameAsync(videoFrame); // Update the UI with prediction UnityEngine.WSA.Application.InvokeOnAppThread(() => { StatusBlock.text = $"Label: {result.PredictionLabel} " + $"Probability: {Math.Round(result.PredictionProbability, 3) * 100}% " + $"Inference time: {result.PredictionTime} ms"; }, false); } catch (Exception ex) { Debug.Log($"Exception {ex}"); } }