private async Task ShowPredictionResults(CustomVisionModelOutput output, double latency)
{
List <Tuple <string, float> > result = output.GetPredictionResult();
Tuple <string, float> topMatch = result?.Where(x => x.Item2 > MinProbabilityValue)?.OrderByDescending(x => x.Item2).FirstOrDefault();
// Update tags in the result panel
if (topMatch != null && topMatch.Item1 != "Negative")
{
if (_lastMatchLabel != topMatch.Item1)
{
_lastMatchLabel = topMatch.Item1;
// if we want to only send alerts when a detected class changes, use this line instead
//IoTHubHelper.Instance.SendDetectedClassAlertToCloudsync(topMatch.Item1, topMatch.Item2);
UpdateSenseHatLights(lightsOn: true);
}
await IoTHubWrapper.Instance.SendDetectedClassAlertToCloudAsync(topMatch.Item1, Math.Round(topMatch.Item2, 2));
await UpdateStatus("Scoring", $"{topMatch.Item1} ({Math.Round(topMatch.Item2 * 100)}%)", $"{Math.Round(1000 / latency)}fps", sendMessageToIoTHub : false);
}
else
{
_lastMatchLabel = null;
UpdateSenseHatLights(lightsOn: false);
await UpdateStatus("Scoring", "Nothing detected", $"{Math.Round(1000 / latency)}fps");
}
}
public async Task ScoreFrame(VideoFrame videoFrame)
{
if (!_isModelLoadedSuccessfully || videoFrame == null)
{
return;
}
try
{
using (SoftwareBitmap bitmapBuffer = new SoftwareBitmap(BitmapPixelFormat.Bgra8,
_customVisionONNXModel.InputImageWidth, _customVisionONNXModel.InputImageHeight, BitmapAlphaMode.Ignore))
{
using (VideoFrame buffer = VideoFrame.CreateWithSoftwareBitmap(bitmapBuffer))
{
await videoFrame.CopyToAsync(buffer);
var input = new CustomVisionModelInput()
{
data = buffer
};
// Perform prediction using ONNX model
DateTime start = DateTime.Now;
CustomVisionModelOutput output = await this._customVisionONNXModel.EvaluateAsync(input);
await ShowPredictionResults(output, Math.Round((DateTime.Now - start).TotalMilliseconds));
}
}
}
catch (Exception ex)
{
this._isModelLoadedSuccessfully = false;
await UpdateStatus("Error", $"Failure scoring camera frame: {ex.Message}");
}
}
private List <Tuple <string, float> > GetModelOutputs(CustomVisionModelOutput output)
{
IList <IDictionary <string, float> > loss = output.loss;
List <Tuple <string, float> > result = new List <Tuple <string, float> >();
foreach (IDictionary <string, float> dict in loss)
{
foreach (var item in dict)
{
result.Add(new Tuple <string, float>(item.Key, item.Value));
}
}
return(result);
}
public async Task ProcessFrame(VideoFrame videoFrame, Canvas visualizationCanvas)
{
if (!isModelLoadedSuccessfully)
{
return;
}
try
{
using (SoftwareBitmap bitmapBuffer = new SoftwareBitmap(BitmapPixelFormat.Bgra8,
CustomVisionModelInputSize, CustomVisionModelInputSize, BitmapAlphaMode.Ignore))
{
using (VideoFrame buffer = VideoFrame.CreateWithSoftwareBitmap(bitmapBuffer))
{
await videoFrame.CopyToAsync(buffer);
var input = new CustomVisionModelInput()
{
data = buffer
};
DateTime start = DateTime.Now;
// Prediction process with ONNX model
CustomVisionModelOutput output = await this.customVisionModel.EvaluateAsync(input);
DateTime end = DateTime.Now;
await this.Dispatcher.RunAsync(Windows.UI.Core.CoreDispatcherPriority.Normal, () =>
{
ShowResults(output);
double predictionTimeInMilliseconds = (end - start).TotalMilliseconds;
this.fpsTextBlock.Text = predictionTimeInMilliseconds > 0 ? $"{Math.Round(1000 / predictionTimeInMilliseconds)} fps" : string.Empty;
});
}
}
}
catch (Exception ex)
{
this.isModelLoadedSuccessfully = false;
await this.Dispatcher.RunAsync(Windows.UI.Core.CoreDispatcherPriority.Normal, async() =>
{
ResetState();
await Util.GenericApiCallExceptionHandler(ex, "Failure processing frame");
});
}
}
private async Task ShowPredictionResults(CustomVisionModelOutput output, double latency)
{
string fpsValue = latency > 0 ? $"{Math.Round(1000 / latency)} fps" : string.Empty;
List <Tuple <string, float> > result = output.GetPredictionResult();
Tuple <string, float> topMatch = result?.Where(x => x.Item2 > MinProbabilityValue)?.OrderByDescending(x => x.Item2).FirstOrDefault();
// Update tags in the result panel
if (topMatch != null && topMatch.Item1 != CustomVisionServiceWrapper.NegativeTag)
{
await UpdateStatus(topMatch.Item1, Math.Round(topMatch.Item2 * 100), fpsValue);
}
else
{
await UpdateStatus("Nothing detected", details : fpsValue);
}
}
public async Task ProcessFrame(VideoFrame videoFrame, Canvas visualizationCanvas)
{
if (customVisionONNXModel == null || videoFrame == null)
{
return;
}
try
{
using (SoftwareBitmap bitmapBuffer = new SoftwareBitmap(BitmapPixelFormat.Bgra8,
customVisionONNXModel.InputImageWidth, customVisionONNXModel.InputImageHeight, BitmapAlphaMode.Ignore))
{
using (VideoFrame buffer = VideoFrame.CreateWithSoftwareBitmap(bitmapBuffer))
{
await videoFrame.CopyToAsync(buffer);
var input = new CustomVisionModelInput()
{
data = buffer
};
DateTime start = DateTime.Now;
// Prediction process with ONNX model
CustomVisionModelOutput output = await this.customVisionONNXModel.EvaluateAsync(input);
await ShowPredictionResults(output, Math.Round((DateTime.Now - start).TotalMilliseconds));
}
}
}
catch (Exception ex)
{
await this.Dispatcher.RunAsync(Windows.UI.Core.CoreDispatcherPriority.Normal, async() =>
{
if (SettingsHelper.Instance.ShowDebugInfo)
{
await Util.GenericApiCallExceptionHandler(ex, "Failure processing frame");
}
});
}
}
private void ShowResults(CustomVisionModelOutput output)
{
List <Tuple <string, float> > result = GetModelOutputs(output);
IEnumerable <Tuple <string, float> > matches = result?.Where(x => x.Item2 > MinProbabilityValue)?.OrderByDescending(x => x.Item2);
// Remove nonexistent tags from the result panel
IEnumerable <string> matchesTagNameCollection = matches?.Select(x => x.Item1);
List <TagScoreControl> nonexistentTagList = this.tagScorePanel.Children?
.Where(x => !matchesTagNameCollection.Contains((string)((UserControl)x).Tag))
.Select(x => x as TagScoreControl)
.ToList();
foreach (TagScoreControl tagItem in nonexistentTagList)
{
this.tagScorePanel.Children?.Remove(tagItem);
}
// Update tags in the result panel
int numClassesOnLastFrame = matches != null?matches.Count() : 0;
if (numClassesOnLastFrame > 0)
{
this.lastMatches = matches;
this.resultsDetails.Visibility = Visibility.Visible;
this.landingMessagePanel.Visibility = Visibility.Collapsed;
int index = 0;
foreach (var item in matches)
{
TagScoreControl tagScoreControl = this.tagScorePanel.Children?.FirstOrDefault(x => (string)((UserControl)x).Tag == item.Item1) as TagScoreControl;
if (tagScoreControl != null)
{
TagScoreViewModel vm = tagScoreControl.DataContext as TagScoreViewModel;
vm.Probability = string.Format("{0}%", Math.Round(item.Item2 * 100));
int filterControlIndex = this.tagScorePanel.Children?.IndexOf(tagScoreControl) ?? -1;
if (filterControlIndex != index)
{
this.tagScorePanel.Children?.Move((uint)filterControlIndex, (uint)index);
}
}
else
{
TagScoreControl newControl = new TagScoreControl
{
Tag = item.Item1,
DataContext = new TagScoreViewModel(item.Item1)
{
Probability = string.Format("{0}%", Math.Round(item.Item2 * 100))
}
};
this.tagScorePanel.Children?.Add(newControl);
}
index++;
}
}
else
{
this.resultsDetails.Visibility = Visibility.Collapsed;
}
}
