private async Task EvaluateFrame(Windows.Media.VideoFrame videoFrame)
{
try
{
var result = await _dnnModel.EvaluateVideoFrameAsync(videoFrame);
if (result.DominantResultProbability > 0)
{
// Further process and surface results to UI on the UI thread
UnityEngine.WSA.Application.InvokeOnAppThread(() =>
{
// Measure distance between user's head and gaze ray hit point => distance to object
var distMessage = string.Empty;
if (_user.GazeHitDistance < 1)
{
distMessage = string.Format("{0:f0} {1}", _user.GazeHitDistance * 100, "centimeter");
}
else
{
distMessage = string.Format("{0:f1} {1}", _user.GazeHitDistance, "meter");
}
// Prepare strings for text and update labels
var deviceKind = ShouldUseGpu ? "GPU" : "CPU";
var labelText = $"Predominant objects detected in {result.ElapsedMilliseconds,3:f0}ms on {deviceKind}\n {result.TopResultsFormatted}";
var speechText = string.Format("This {0} a {1} {2} in front of you",
result.DominantResultProbability > ProbabilityThreshold ? "is likely" : "might be",
result.DominantResultLabel,
distMessage);
StatusBlock.text = labelText;
// Check if the previous result was the same and only progress further if not to avoid a loop of same audio
if (!_tts.IsSpeaking() && result.DominantResultLabel != _previousDominantResult)
{
_tts.StartSpeaking(speechText);
_previousDominantResult = result.DominantResultLabel;
}
}, false);
}
}
catch (Exception ex)
{
//IsRunning = false;
UnityEngine.WSA.Application.InvokeOnAppThread(() =>
{
//StatusBlock.text = $"Error loop: {ex.Message}";
//Debug.LogError(ex);
//Debug.LogError(videoFrame.Direct3DSurface == null ? "D3D null" : "D3D set");
//if (videoFrame.Direct3DSurface != null)
//{
// Debug.LogError(videoFrame.Direct3DSurface.Description.Format);
// Debug.LogError(videoFrame.Direct3DSurface.Description.Width);
// Debug.LogError(videoFrame.Direct3DSurface.Description.Height);
//}
}, false);
}
}
private async Task EvaluateFrame(Windows.Media.VideoFrame videoFrame)
{
try
{
var result = await _dnnModel.PredictImageAsync(videoFrame);
var first = result.FirstOrDefault();
if (first != null && first.Probability > .5)
{
// Further process and surface results to UI on the UI thread
UnityEngine.WSA.Application.InvokeOnAppThread(() =>
{
// Measure distance between user's head and gaze ray hit point => distance to object
var distMessage = string.Empty;
if (_user.GazeHitDistance < 1)
{
distMessage = string.Format("{0:f0} {1}", _user.GazeHitDistance * 100, "centimeter");
}
else
{
distMessage = string.Format("{0:f1} {1}", _user.GazeHitDistance, "meter");
}
// Prepare strings for text and update labels
var labelText = $"Found: {first.TagName}";
StatusBlock.text = labelText;
}, false);
}
}
catch (Exception ex)
{
//IsRunning = false;
UnityEngine.WSA.Application.InvokeOnAppThread(() =>
{
//StatusBlock.text = $"Error loop: {ex.Message}";
//Debug.LogError(ex);
//Debug.LogError(videoFrame.Direct3DSurface == null ? "D3D null" : "D3D set");
//if (videoFrame.Direct3DSurface != null)
//{
// Debug.LogError(videoFrame.Direct3DSurface.Description.Format);
// Debug.LogError(videoFrame.Direct3DSurface.Description.Width);
// Debug.LogError(videoFrame.Direct3DSurface.Description.Height);
//}
}, false);
}
}
private async Task EvaluateFrame(Windows.Media.VideoFrame videoFrame)
{
try
{
// Get the current network prediction from model and input frame
var result = await _networkModel.EvaluateVideoFrameAsync(videoFrame);
// Update the UI with prediction
UnityEngine.WSA.Application.InvokeOnAppThread(() =>
{
StatusBlock.text = $"Label: {result.PredictionLabel} " +
$"Probability: {Math.Round(result.PredictionProbability, 3) * 100}% " +
$"Inference time: {result.PredictionTime} ms";
}, false);
}
catch (Exception ex)
{
Debug.Log($"Exception {ex}");
}
}