private async void Current_SoftwareBitmapFrameCaptured(object sender, SoftwareBitmapEventArgs e) { Debug.WriteLine("FrameCaptured"); Debug.WriteLine($"Frame evaluation started {DateTime.Now}"); if (e.SoftwareBitmap != null) { BitmapPixelFormat bpf = e.SoftwareBitmap.BitmapPixelFormat; var uncroppedBitmap = SoftwareBitmap.Convert(e.SoftwareBitmap, BitmapPixelFormat.Nv12); var faces = await _faceDetector.DetectFacesAsync(uncroppedBitmap); if (faces.Count > 0) { //crop image to focus on face portion var faceBox = faces[0].FaceBox; VideoFrame inputFrame = VideoFrame.CreateWithSoftwareBitmap(e.SoftwareBitmap); VideoFrame tmp = null; tmp = new VideoFrame(e.SoftwareBitmap.BitmapPixelFormat, (int)(faceBox.Width + faceBox.Width % 2) - 2, (int)(faceBox.Height + faceBox.Height % 2) - 2); await inputFrame.CopyToAsync(tmp, faceBox, null); //crop image to fit model input requirements VideoFrame croppedInputImage = new VideoFrame(BitmapPixelFormat.Gray8, (int)_inputImageDescriptor.Shape[3], (int)_inputImageDescriptor.Shape[2]); var srcBounds = GetCropBounds( tmp.SoftwareBitmap.PixelWidth, tmp.SoftwareBitmap.PixelHeight, croppedInputImage.SoftwareBitmap.PixelWidth, croppedInputImage.SoftwareBitmap.PixelHeight); await tmp.CopyToAsync(croppedInputImage, srcBounds, null); ImageFeatureValue imageTensor = ImageFeatureValue.CreateFromVideoFrame(croppedInputImage); _binding = new LearningModelBinding(_session); TensorFloat outputTensor = TensorFloat.Create(_outputTensorDescriptor.Shape); List <float> _outputVariableList = new List <float>(); // Bind inputs + outputs _binding.Bind(_inputImageDescriptor.Name, imageTensor); _binding.Bind(_outputTensorDescriptor.Name, outputTensor); // Evaluate results var results = await _session.EvaluateAsync(_binding, new Guid().ToString()); Debug.WriteLine("ResultsEvaluated: " + results.ToString()); var outputTensorList = outputTensor.GetAsVectorView(); var resultsList = new List <float>(outputTensorList.Count); for (int i = 0; i < outputTensorList.Count; i++) { resultsList.Add(outputTensorList[i]); } var softMaxexOutputs = SoftMax(resultsList); double maxProb = 0; int maxIndex = 0; // Comb through the evaluation results for (int i = 0; i < Constants.POTENTIAL_EMOJI_NAME_LIST.Count(); i++) { // Record the dominant emotion probability & its location if (softMaxexOutputs[i] > maxProb) { maxIndex = i; maxProb = softMaxexOutputs[i]; } //for evaluations run on the EmotionPage, record info about single specific emotion of interest if (CurrentEmojis._currentEmoji != null && Constants.POTENTIAL_EMOJI_NAME_LIST[i].Equals(CurrentEmojis._currentEmoji.Name)) { SoftwareBitmap potentialBestPic; try { potentialBestPic = SoftwareBitmap.Convert(uncroppedBitmap, BitmapPixelFormat.Bgra8); } catch (Exception ex) { Debug.WriteLine($"Error converting SoftwareBitmap. Details:{ex.Message}. Attempting to continue..."); return; } await Windows.ApplicationModel.Core.CoreApplication.MainView.CoreWindow.Dispatcher.RunAsync(CoreDispatcherPriority.Normal, async() => { // Give user immediate visual feedback by updating success gauge ScoreUpdated?.Invoke(this, new EmotionPageGaugeScoreEventArgs() { Score = softMaxexOutputs[i] }); // Save original pic for each emotion no matter how bad it is (and record its associated info) double bestScore = CurrentEmojis._emojis.Emojis[CurrentEmojis._currentEmojiIndex].BestScore; if (softMaxexOutputs[i] > bestScore) { CurrentEmojis._emojis.Emojis[CurrentEmojis._currentEmojiIndex].BestScore = softMaxexOutputs[i]; var source = new SoftwareBitmapSource(); await source.SetBitmapAsync(potentialBestPic); // Create format of potentialBestPic to be displayed in a gif later SoftwareBitmap tmpBitmap = potentialBestPic; WriteableBitmap wb = new WriteableBitmap(tmpBitmap.PixelWidth, tmpBitmap.PixelHeight); tmpBitmap.CopyToBuffer(wb.PixelBuffer); CurrentEmojis._emojis.Emojis[CurrentEmojis._currentEmojiIndex].BestPic = source; CurrentEmojis._emojis.Emojis[CurrentEmojis._currentEmojiIndex].ShowOopsIcon = false; CurrentEmojis._emojis.Emojis[CurrentEmojis._currentEmojiIndex].BestPicWB = wb; } } ); } } Debug.WriteLine($"Probability = {maxProb}, Threshold set to = {Constants.CLASSIFICATION_CERTAINTY_THRESHOLD}, Emotion = {Constants.POTENTIAL_EMOJI_NAME_LIST[maxIndex]}"); // For evaluations run on the MainPage, update the emoji carousel if (maxProb >= Constants.CLASSIFICATION_CERTAINTY_THRESHOLD) { Debug.WriteLine("first page emoji should start to update"); IntelligenceServiceEmotionClassified?.Invoke(this, new ClassifiedEmojiEventArgs(CurrentEmojis._emojis.Emojis[maxIndex])); } // Dispose of resources if (e.SoftwareBitmap != null) { e.SoftwareBitmap.Dispose(); e.SoftwareBitmap = null; } } } IntelligenceServiceProcessingCompleted?.Invoke(this, null); Debug.WriteLine($"Frame evaluation finished {DateTime.Now}"); }
private async void Current_SoftwareBitmapFrameCaptured(object sender, SoftwareBitmapEventArgs e) { Debug.WriteLine("FrameCaptured"); Debug.WriteLine($"Frame evaluation started {DateTime.Now}"); if (e.SoftwareBitmap != null) { BitmapPixelFormat bpf = e.SoftwareBitmap.BitmapPixelFormat; var uncroppedBitmap = SoftwareBitmap.Convert(e.SoftwareBitmap, BitmapPixelFormat.Nv12); var faces = await _faceDetector.DetectFacesAsync(uncroppedBitmap); if (faces.Count > 0) { //crop image to focus on face portion var faceBox = faces[0].FaceBox; VideoFrame inputFrame = VideoFrame.CreateWithSoftwareBitmap(e.SoftwareBitmap); VideoFrame tmp = null; tmp = new VideoFrame(e.SoftwareBitmap.BitmapPixelFormat, (int)(faceBox.Width + faceBox.Width % 2) - 2, (int)(faceBox.Height + faceBox.Height % 2) - 2); await inputFrame.CopyToAsync(tmp, faceBox, null); //crop image to fit model input requirements VideoFrame croppedInputImage = new VideoFrame(BitmapPixelFormat.Gray8, (int)_inputImageDescriptor.Shape[3], (int)_inputImageDescriptor.Shape[2]); var srcBounds = GetCropBounds( tmp.SoftwareBitmap.PixelWidth, tmp.SoftwareBitmap.PixelHeight, croppedInputImage.SoftwareBitmap.PixelWidth, croppedInputImage.SoftwareBitmap.PixelHeight); await tmp.CopyToAsync(croppedInputImage, srcBounds, null); ImageFeatureValue imageTensor = ImageFeatureValue.CreateFromVideoFrame(croppedInputImage); _binding = new LearningModelBinding(_session); TensorFloat outputTensor = TensorFloat.Create(_outputTensorDescriptor.Shape); List <float> _outputVariableList = new List <float>(); // Bind inputs + outputs _binding.Bind(_inputImageDescriptor.Name, imageTensor); _binding.Bind(_outputTensorDescriptor.Name, outputTensor); // Evaluate results var results = await _session.EvaluateAsync(_binding, new Guid().ToString()); Debug.WriteLine("ResultsEvaluated: " + results.ToString()); var outputTensorList = outputTensor.GetAsVectorView(); var resultsList = new List <float>(outputTensorList.Count); for (int i = 0; i < outputTensorList.Count; i++) { resultsList.Add(outputTensorList[i]); } var softMaxexOutputs = SoftMax(resultsList); double maxProb = 0; int maxIndex = 0; // Comb through the evaluation results for (int i = 0; i < Constants.POTENTIAL_EMOJI_NAME_LIST.Count(); i++) { // Record the dominant emotion probability & its location if (softMaxexOutputs[i] > maxProb) { maxIndex = i; maxProb = softMaxexOutputs[i]; } } Debug.WriteLine($"Probability = {maxProb}, Threshold set to = {Constants.CLASSIFICATION_CERTAINTY_THRESHOLD}, Emotion = {Constants.POTENTIAL_EMOJI_NAME_LIST[maxIndex]}"); // For evaluations run on the MainPage, update the emoji carousel if (maxProb >= Constants.CLASSIFICATION_CERTAINTY_THRESHOLD) { Debug.WriteLine("first page emoji should start to update"); IntelligenceServiceEmotionClassified?.Invoke(this, new ClassifiedEmojiEventArgs(CurrentEmojis._emojis.Emojis[maxIndex])); } // Dispose of resources if (e.SoftwareBitmap != null) { e.SoftwareBitmap.Dispose(); e.SoftwareBitmap = null; } } } IntelligenceServiceProcessingCompleted?.Invoke(this, null); Debug.WriteLine($"Frame evaluation finished {DateTime.Now}"); }