private async Task ShowPredictionResults(CustomVisionModelOutput output, double latency) { List <Tuple <string, float> > result = output.GetPredictionResult(); Tuple <string, float> topMatch = result?.Where(x => x.Item2 > MinProbabilityValue)?.OrderByDescending(x => x.Item2).FirstOrDefault(); // Update tags in the result panel if (topMatch != null && topMatch.Item1 != "Negative") { if (_lastMatchLabel != topMatch.Item1) { _lastMatchLabel = topMatch.Item1; // if we want to only send alerts when a detected class changes, use this line instead //IoTHubHelper.Instance.SendDetectedClassAlertToCloudsync(topMatch.Item1, topMatch.Item2); UpdateSenseHatLights(lightsOn: true); } await IoTHubWrapper.Instance.SendDetectedClassAlertToCloudAsync(topMatch.Item1, Math.Round(topMatch.Item2, 2)); await UpdateStatus("Scoring", $"{topMatch.Item1} ({Math.Round(topMatch.Item2 * 100)}%)", $"{Math.Round(1000 / latency)}fps", sendMessageToIoTHub : false); } else { _lastMatchLabel = null; UpdateSenseHatLights(lightsOn: false); await UpdateStatus("Scoring", "Nothing detected", $"{Math.Round(1000 / latency)}fps"); } }
public async Task ScoreFrame(VideoFrame videoFrame) { if (!_isModelLoadedSuccessfully || videoFrame == null) { return; } try { using (SoftwareBitmap bitmapBuffer = new SoftwareBitmap(BitmapPixelFormat.Bgra8, _customVisionONNXModel.InputImageWidth, _customVisionONNXModel.InputImageHeight, BitmapAlphaMode.Ignore)) { using (VideoFrame buffer = VideoFrame.CreateWithSoftwareBitmap(bitmapBuffer)) { await videoFrame.CopyToAsync(buffer); var input = new CustomVisionModelInput() { data = buffer }; // Perform prediction using ONNX model DateTime start = DateTime.Now; CustomVisionModelOutput output = await this._customVisionONNXModel.EvaluateAsync(input); await ShowPredictionResults(output, Math.Round((DateTime.Now - start).TotalMilliseconds)); } } } catch (Exception ex) { this._isModelLoadedSuccessfully = false; await UpdateStatus("Error", $"Failure scoring camera frame: {ex.Message}"); } }
private List <Tuple <string, float> > GetModelOutputs(CustomVisionModelOutput output) { IList <IDictionary <string, float> > loss = output.loss; List <Tuple <string, float> > result = new List <Tuple <string, float> >(); foreach (IDictionary <string, float> dict in loss) { foreach (var item in dict) { result.Add(new Tuple <string, float>(item.Key, item.Value)); } } return(result); }
public async Task ProcessFrame(VideoFrame videoFrame, Canvas visualizationCanvas) { if (!isModelLoadedSuccessfully) { return; } try { using (SoftwareBitmap bitmapBuffer = new SoftwareBitmap(BitmapPixelFormat.Bgra8, CustomVisionModelInputSize, CustomVisionModelInputSize, BitmapAlphaMode.Ignore)) { using (VideoFrame buffer = VideoFrame.CreateWithSoftwareBitmap(bitmapBuffer)) { await videoFrame.CopyToAsync(buffer); var input = new CustomVisionModelInput() { data = buffer }; DateTime start = DateTime.Now; // Prediction process with ONNX model CustomVisionModelOutput output = await this.customVisionModel.EvaluateAsync(input); DateTime end = DateTime.Now; await this.Dispatcher.RunAsync(Windows.UI.Core.CoreDispatcherPriority.Normal, () => { ShowResults(output); double predictionTimeInMilliseconds = (end - start).TotalMilliseconds; this.fpsTextBlock.Text = predictionTimeInMilliseconds > 0 ? $"{Math.Round(1000 / predictionTimeInMilliseconds)} fps" : string.Empty; }); } } } catch (Exception ex) { this.isModelLoadedSuccessfully = false; await this.Dispatcher.RunAsync(Windows.UI.Core.CoreDispatcherPriority.Normal, async() => { ResetState(); await Util.GenericApiCallExceptionHandler(ex, "Failure processing frame"); }); } }
private async Task ShowPredictionResults(CustomVisionModelOutput output, double latency) { string fpsValue = latency > 0 ? $"{Math.Round(1000 / latency)} fps" : string.Empty; List <Tuple <string, float> > result = output.GetPredictionResult(); Tuple <string, float> topMatch = result?.Where(x => x.Item2 > MinProbabilityValue)?.OrderByDescending(x => x.Item2).FirstOrDefault(); // Update tags in the result panel if (topMatch != null && topMatch.Item1 != CustomVisionServiceWrapper.NegativeTag) { await UpdateStatus(topMatch.Item1, Math.Round(topMatch.Item2 * 100), fpsValue); } else { await UpdateStatus("Nothing detected", details : fpsValue); } }
public async Task ProcessFrame(VideoFrame videoFrame, Canvas visualizationCanvas) { if (customVisionONNXModel == null || videoFrame == null) { return; } try { using (SoftwareBitmap bitmapBuffer = new SoftwareBitmap(BitmapPixelFormat.Bgra8, customVisionONNXModel.InputImageWidth, customVisionONNXModel.InputImageHeight, BitmapAlphaMode.Ignore)) { using (VideoFrame buffer = VideoFrame.CreateWithSoftwareBitmap(bitmapBuffer)) { await videoFrame.CopyToAsync(buffer); var input = new CustomVisionModelInput() { data = buffer }; DateTime start = DateTime.Now; // Prediction process with ONNX model CustomVisionModelOutput output = await this.customVisionONNXModel.EvaluateAsync(input); await ShowPredictionResults(output, Math.Round((DateTime.Now - start).TotalMilliseconds)); } } } catch (Exception ex) { await this.Dispatcher.RunAsync(Windows.UI.Core.CoreDispatcherPriority.Normal, async() => { if (SettingsHelper.Instance.ShowDebugInfo) { await Util.GenericApiCallExceptionHandler(ex, "Failure processing frame"); } }); } }
private void ShowResults(CustomVisionModelOutput output) { List <Tuple <string, float> > result = GetModelOutputs(output); IEnumerable <Tuple <string, float> > matches = result?.Where(x => x.Item2 > MinProbabilityValue)?.OrderByDescending(x => x.Item2); // Remove nonexistent tags from the result panel IEnumerable <string> matchesTagNameCollection = matches?.Select(x => x.Item1); List <TagScoreControl> nonexistentTagList = this.tagScorePanel.Children? .Where(x => !matchesTagNameCollection.Contains((string)((UserControl)x).Tag)) .Select(x => x as TagScoreControl) .ToList(); foreach (TagScoreControl tagItem in nonexistentTagList) { this.tagScorePanel.Children?.Remove(tagItem); } // Update tags in the result panel int numClassesOnLastFrame = matches != null?matches.Count() : 0; if (numClassesOnLastFrame > 0) { this.lastMatches = matches; this.resultsDetails.Visibility = Visibility.Visible; this.landingMessagePanel.Visibility = Visibility.Collapsed; int index = 0; foreach (var item in matches) { TagScoreControl tagScoreControl = this.tagScorePanel.Children?.FirstOrDefault(x => (string)((UserControl)x).Tag == item.Item1) as TagScoreControl; if (tagScoreControl != null) { TagScoreViewModel vm = tagScoreControl.DataContext as TagScoreViewModel; vm.Probability = string.Format("{0}%", Math.Round(item.Item2 * 100)); int filterControlIndex = this.tagScorePanel.Children?.IndexOf(tagScoreControl) ?? -1; if (filterControlIndex != index) { this.tagScorePanel.Children?.Move((uint)filterControlIndex, (uint)index); } } else { TagScoreControl newControl = new TagScoreControl { Tag = item.Item1, DataContext = new TagScoreViewModel(item.Item1) { Probability = string.Format("{0}%", Math.Round(item.Item2 * 100)) } }; this.tagScorePanel.Children?.Add(newControl); } index++; } } else { this.resultsDetails.Visibility = Visibility.Collapsed; } }