private async Task LoadAndEvaluateModelAsync(VideoFrame _inputFrame, string _modelFileName) { LearningModelBinding _binding = null; VideoFrame _outputFrame = null; LearningModelSession _session; try { //Load and create the model if (_model == null) { var modelFile = await StorageFile.GetFileFromApplicationUriAsync(new Uri($"ms-appx:///{_modelFileName}")); _model = await LearningModel.LoadFromStorageFileAsync(modelFile); } // Create the evaluation session with the model _session = new LearningModelSession(_model); // Get input and output features of the model var inputFeatures = _model.InputFeatures.ToList(); var outputFeatures = _model.OutputFeatures.ToList(); // Create binding and then bind input/ output features _binding = new LearningModelBinding(_session); _inputImageDescriptor = inputFeatures.FirstOrDefault(feature => feature.Kind == LearningModelFeatureKind.Tensor) as TensorFeatureDescriptor; _outputTensorDescriptor = outputFeatures.FirstOrDefault(feature => feature.Kind == LearningModelFeatureKind.Tensor) as TensorFeatureDescriptor; TensorFloat outputTensor = TensorFloat.Create(_outputTensorDescriptor.Shape); ImageFeatureValue imageTensor = ImageFeatureValue.CreateFromVideoFrame(_inputFrame); // Bind inputs +outputs _binding.Bind(_inputImageDescriptor.Name, imageTensor); _binding.Bind(_outputTensorDescriptor.Name, outputTensor); // Evaluate and get the results var results = await _session.EvaluateAsync(_binding, "test"); Debug.WriteLine("ResultsEvaluated: " + results.ToString()); var outputTensorList = outputTensor.GetAsVectorView(); var resultsList = new List <float>(outputTensorList.Count); for (int i = 0; i < outputTensorList.Count; i++) { resultsList.Add(outputTensorList[i]); } } catch (Exception ex) { Debug.WriteLine($"error: {ex.Message}"); _model = null; } }
public async Task <TensorFloat> EvaluateAsync(TensorFloat input) { binding.Bind(inName, input); binding.Bind(outName, TensorFloat.Create(new long[] { 1, 1, InHeight, OutHeight })); var result = await session.EvaluateAsync(binding, inName); return(result.Outputs[outName] as TensorFloat); }
private void SampleInputsGridView_SelectionChanged(object sender, SelectionChangedEventArgs e) { var gridView = sender as GridView; var thumbnail = gridView.SelectedItem as WinMLSamplesGallery.Controls.Thumbnail; if (thumbnail != null) { var image = thumbnail.ImageUri; var file = StorageFile.GetFileFromApplicationUriAsync(new Uri(image)).GetAwaiter().GetResult(); var softwareBitmap = CreateSoftwareBitmapFromStorageFile(file); tensorizationSession_ = CreateLearningModelSession( TensorizationModels.ReshapeFlatBufferNHWC( 1, 4, softwareBitmap.PixelHeight, softwareBitmap.PixelWidth, 416, 416)); // Tensorize var stream = file.OpenAsync(FileAccessMode.Read).GetAwaiter().GetResult(); var decoder = BitmapDecoder.CreateAsync(stream).GetAwaiter().GetResult(); var bitmap = decoder.GetSoftwareBitmapAsync(BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied).GetAwaiter().GetResult(); var pixelDataProvider = decoder.GetPixelDataAsync().GetAwaiter().GetResult(); var bytes = pixelDataProvider.DetachPixelData(); var buffer = bytes.AsBuffer(); // Does this do a copy?? var inputRawTensor = TensorUInt8Bit.CreateFromBuffer(new long[] { 1, buffer.Length }, buffer); // 3 channel NCHW var tensorizeOutput = TensorFloat.Create(new long[] { 1, 416, 416, 3 }); var b = new LearningModelBinding(tensorizationSession_); b.Bind(tensorizationSession_.Model.InputFeatures[0].Name, inputRawTensor); b.Bind(tensorizationSession_.Model.OutputFeatures[0].Name, tensorizeOutput); tensorizationSession_.Evaluate(b, ""); // Resize var resizeBinding = new LearningModelBinding(_session); resizeBinding.Bind(_session.Model.InputFeatures[0].Name, tensorizeOutput); var results = _session.Evaluate(resizeBinding, ""); var output1 = results.Output(0) as TensorFloat; var data = output1.GetAsVectorView(); var detections = ParseResult(data.ToList <float>().ToArray()); Comparer cp = new Comparer(); detections.Sort(cp); var final = NMS(detections); RenderImageInMainPanel(softwareBitmap); } }
private static LearningModelEvaluationResult Evaluate(LearningModelSession session, object input) { // Create the binding var binding = new LearningModelBinding(session); // Create an empty output, that will keep the output resources on the GPU // It will be chained into a the post processing on the GPU as well var output = TensorFloat.Create(); // Bind inputs and outputs // For squeezenet these evaluate to "data", and "squeezenet0_flatten0_reshape0" string inputName = session.Model.InputFeatures[0].Name; string outputName = session.Model.OutputFeatures[0].Name; binding.Bind(inputName, input); var outputBindProperties = new PropertySet(); outputBindProperties.Add("DisableTensorCpuSync", PropertyValue.CreateBoolean(true)); binding.Bind(outputName, output, outputBindProperties); // Evaluate return(session.Evaluate(binding, "")); }
/// <summary> /// Evaluate the VideoFrame passed in as arg /// </summary> /// <param name="inputFrame"></param> /// <returns></returns> private async Task EvaluateVideoFrameAsync(VideoFrame inputFrame) { if (inputFrame != null) { try { StatusBlock.Text = "Binding image..."; // create a binding object from the session LearningModelBinding binding = new LearningModelBinding(_session); // bind the input image ImageFeatureValue imageTensor = ImageFeatureValue.CreateFromVideoFrame(inputFrame); binding.Bind("data_0", imageTensor); // temp: there is a bug where winml doesn't allow unbound outputs yet, prebind the output! { TensorFeatureDescriptor outputTensorDescription = _model.OutputFeatures.FirstOrDefault( feature => feature.Name == "softmaxout_1" ) as TensorFeatureDescriptor; TensorFloat outputTensor = TensorFloat.Create(outputTensorDescription.Shape); binding.Bind("softmaxout_1", outputTensor); } StatusBlock.Text = "Running model..."; int ticks = Environment.TickCount; // Process the frame with the model var results = await _session.EvaluateAsync(binding, $"Run { ++_runCount } "); ticks = Environment.TickCount - ticks; // retrieve results from evaluation var resultTensor = results.Outputs["softmaxout_1"] as TensorFloat; var resultVector = resultTensor.GetAsVectorView(); // Find the top 3 probabilities List <float> topProbabilities = new List <float>() { 0.0f, 0.0f, 0.0f }; List <int> topProbabilityLabelIndexes = new List <int>() { 0, 0, 0 }; // SqueezeNet returns a list of 1000 options, with probabilities for each, loop through all for (int i = 0; i < resultVector.Count(); i++) { // is it one of the top 3? for (int j = 0; j < 3; j++) { if (resultVector[i] > topProbabilities[j]) { topProbabilityLabelIndexes[j] = i; topProbabilities[j] = resultVector[i]; break; } } } // Display the result string message = $"Run took { ticks } ticks"; for (int i = 0; i < 3; i++) { message += $"\n\"{ _labels[topProbabilityLabelIndexes[i]]}\" with confidence of { topProbabilities[i]}"; } StatusBlock.Text = message; } catch (Exception ex) { StatusBlock.Text = $"error: {ex.Message}"; } ButtonRun.IsEnabled = true; } }
private async void Current_SoftwareBitmapFrameCaptured(object sender, SoftwareBitmapEventArgs e) { Debug.WriteLine("FrameCaptured"); Debug.WriteLine($"Frame evaluation started {DateTime.Now}"); if (e.SoftwareBitmap != null) { BitmapPixelFormat bpf = e.SoftwareBitmap.BitmapPixelFormat; var uncroppedBitmap = SoftwareBitmap.Convert(e.SoftwareBitmap, BitmapPixelFormat.Nv12); var faces = await _faceDetector.DetectFacesAsync(uncroppedBitmap); if (faces.Count > 0) { //crop image to focus on face portion var faceBox = faces[0].FaceBox; VideoFrame inputFrame = VideoFrame.CreateWithSoftwareBitmap(e.SoftwareBitmap); VideoFrame tmp = null; tmp = new VideoFrame(e.SoftwareBitmap.BitmapPixelFormat, (int)(faceBox.Width + faceBox.Width % 2) - 2, (int)(faceBox.Height + faceBox.Height % 2) - 2); await inputFrame.CopyToAsync(tmp, faceBox, null); //crop image to fit model input requirements VideoFrame croppedInputImage = new VideoFrame(BitmapPixelFormat.Gray8, (int)_inputImageDescriptor.Shape[3], (int)_inputImageDescriptor.Shape[2]); var srcBounds = GetCropBounds( tmp.SoftwareBitmap.PixelWidth, tmp.SoftwareBitmap.PixelHeight, croppedInputImage.SoftwareBitmap.PixelWidth, croppedInputImage.SoftwareBitmap.PixelHeight); await tmp.CopyToAsync(croppedInputImage, srcBounds, null); ImageFeatureValue imageTensor = ImageFeatureValue.CreateFromVideoFrame(croppedInputImage); _binding = new LearningModelBinding(_session); TensorFloat outputTensor = TensorFloat.Create(_outputTensorDescriptor.Shape); List <float> _outputVariableList = new List <float>(); // Bind inputs + outputs _binding.Bind(_inputImageDescriptor.Name, imageTensor); _binding.Bind(_outputTensorDescriptor.Name, outputTensor); // Evaluate results var results = await _session.EvaluateAsync(_binding, new Guid().ToString()); Debug.WriteLine("ResultsEvaluated: " + results.ToString()); var outputTensorList = outputTensor.GetAsVectorView(); var resultsList = new List <float>(outputTensorList.Count); for (int i = 0; i < outputTensorList.Count; i++) { resultsList.Add(outputTensorList[i]); } var softMaxexOutputs = SoftMax(resultsList); double maxProb = 0; int maxIndex = 0; // Comb through the evaluation results for (int i = 0; i < Constants.POTENTIAL_EMOJI_NAME_LIST.Count(); i++) { // Record the dominant emotion probability & its location if (softMaxexOutputs[i] > maxProb) { maxIndex = i; maxProb = softMaxexOutputs[i]; } //for evaluations run on the EmotionPage, record info about single specific emotion of interest if (CurrentEmojis._currentEmoji != null && Constants.POTENTIAL_EMOJI_NAME_LIST[i].Equals(CurrentEmojis._currentEmoji.Name)) { SoftwareBitmap potentialBestPic; try { potentialBestPic = SoftwareBitmap.Convert(uncroppedBitmap, BitmapPixelFormat.Bgra8); } catch (Exception ex) { Debug.WriteLine($"Error converting SoftwareBitmap. Details:{ex.Message}. Attempting to continue..."); return; } await Windows.ApplicationModel.Core.CoreApplication.MainView.CoreWindow.Dispatcher.RunAsync(CoreDispatcherPriority.Normal, async() => { // Give user immediate visual feedback by updating success gauge ScoreUpdated?.Invoke(this, new EmotionPageGaugeScoreEventArgs() { Score = softMaxexOutputs[i] }); // Save original pic for each emotion no matter how bad it is (and record its associated info) double bestScore = CurrentEmojis._emojis.Emojis[CurrentEmojis._currentEmojiIndex].BestScore; if (softMaxexOutputs[i] > bestScore) { CurrentEmojis._emojis.Emojis[CurrentEmojis._currentEmojiIndex].BestScore = softMaxexOutputs[i]; var source = new SoftwareBitmapSource(); await source.SetBitmapAsync(potentialBestPic); // Create format of potentialBestPic to be displayed in a gif later SoftwareBitmap tmpBitmap = potentialBestPic; WriteableBitmap wb = new WriteableBitmap(tmpBitmap.PixelWidth, tmpBitmap.PixelHeight); tmpBitmap.CopyToBuffer(wb.PixelBuffer); CurrentEmojis._emojis.Emojis[CurrentEmojis._currentEmojiIndex].BestPic = source; CurrentEmojis._emojis.Emojis[CurrentEmojis._currentEmojiIndex].ShowOopsIcon = false; CurrentEmojis._emojis.Emojis[CurrentEmojis._currentEmojiIndex].BestPicWB = wb; } } ); } } Debug.WriteLine($"Probability = {maxProb}, Threshold set to = {Constants.CLASSIFICATION_CERTAINTY_THRESHOLD}, Emotion = {Constants.POTENTIAL_EMOJI_NAME_LIST[maxIndex]}"); // For evaluations run on the MainPage, update the emoji carousel if (maxProb >= Constants.CLASSIFICATION_CERTAINTY_THRESHOLD) { Debug.WriteLine("first page emoji should start to update"); IntelligenceServiceEmotionClassified?.Invoke(this, new ClassifiedEmojiEventArgs(CurrentEmojis._emojis.Emojis[maxIndex])); } // Dispose of resources if (e.SoftwareBitmap != null) { e.SoftwareBitmap.Dispose(); e.SoftwareBitmap = null; } } } IntelligenceServiceProcessingCompleted?.Invoke(this, null); Debug.WriteLine($"Frame evaluation finished {DateTime.Now}"); }
private async void Current_SoftwareBitmapFrameCaptured(object sender, SoftwareBitmapEventArgs e) { Debug.WriteLine("FrameCaptured"); Debug.WriteLine($"Frame evaluation started {DateTime.Now}"); if (e.SoftwareBitmap != null) { BitmapPixelFormat bpf = e.SoftwareBitmap.BitmapPixelFormat; var uncroppedBitmap = SoftwareBitmap.Convert(e.SoftwareBitmap, BitmapPixelFormat.Nv12); var faces = await _faceDetector.DetectFacesAsync(uncroppedBitmap); if (faces.Count > 0) { //crop image to focus on face portion var faceBox = faces[0].FaceBox; VideoFrame inputFrame = VideoFrame.CreateWithSoftwareBitmap(e.SoftwareBitmap); VideoFrame tmp = null; tmp = new VideoFrame(e.SoftwareBitmap.BitmapPixelFormat, (int)(faceBox.Width + faceBox.Width % 2) - 2, (int)(faceBox.Height + faceBox.Height % 2) - 2); await inputFrame.CopyToAsync(tmp, faceBox, null); //crop image to fit model input requirements VideoFrame croppedInputImage = new VideoFrame(BitmapPixelFormat.Gray8, (int)_inputImageDescriptor.Shape[3], (int)_inputImageDescriptor.Shape[2]); var srcBounds = GetCropBounds( tmp.SoftwareBitmap.PixelWidth, tmp.SoftwareBitmap.PixelHeight, croppedInputImage.SoftwareBitmap.PixelWidth, croppedInputImage.SoftwareBitmap.PixelHeight); await tmp.CopyToAsync(croppedInputImage, srcBounds, null); ImageFeatureValue imageTensor = ImageFeatureValue.CreateFromVideoFrame(croppedInputImage); _binding = new LearningModelBinding(_session); TensorFloat outputTensor = TensorFloat.Create(_outputTensorDescriptor.Shape); List <float> _outputVariableList = new List <float>(); // Bind inputs + outputs _binding.Bind(_inputImageDescriptor.Name, imageTensor); _binding.Bind(_outputTensorDescriptor.Name, outputTensor); // Evaluate results var results = await _session.EvaluateAsync(_binding, new Guid().ToString()); Debug.WriteLine("ResultsEvaluated: " + results.ToString()); var outputTensorList = outputTensor.GetAsVectorView(); var resultsList = new List <float>(outputTensorList.Count); for (int i = 0; i < outputTensorList.Count; i++) { resultsList.Add(outputTensorList[i]); } var softMaxexOutputs = SoftMax(resultsList); double maxProb = 0; int maxIndex = 0; // Comb through the evaluation results for (int i = 0; i < Constants.POTENTIAL_EMOJI_NAME_LIST.Count(); i++) { // Record the dominant emotion probability & its location if (softMaxexOutputs[i] > maxProb) { maxIndex = i; maxProb = softMaxexOutputs[i]; } } Debug.WriteLine($"Probability = {maxProb}, Threshold set to = {Constants.CLASSIFICATION_CERTAINTY_THRESHOLD}, Emotion = {Constants.POTENTIAL_EMOJI_NAME_LIST[maxIndex]}"); // For evaluations run on the MainPage, update the emoji carousel if (maxProb >= Constants.CLASSIFICATION_CERTAINTY_THRESHOLD) { Debug.WriteLine("first page emoji should start to update"); IntelligenceServiceEmotionClassified?.Invoke(this, new ClassifiedEmojiEventArgs(CurrentEmojis._emojis.Emojis[maxIndex])); } // Dispose of resources if (e.SoftwareBitmap != null) { e.SoftwareBitmap.Dispose(); e.SoftwareBitmap = null; } } } IntelligenceServiceProcessingCompleted?.Invoke(this, null); Debug.WriteLine($"Frame evaluation finished {DateTime.Now}"); }
unsafe private TensorFloat CustomTensorize(List <VideoFrame> frameList, List <float> mean, List <float> std, bool toRGB = false) { int temp_len = frameList.Count(); SoftwareBitmap softwareBitmap = frameList[0].SoftwareBitmap; Int32 height = softwareBitmap.PixelHeight; Int32 width = softwareBitmap.PixelWidth; BitmapPixelFormat pixelFormat = softwareBitmap.BitmapPixelFormat; Int32 channels = BitmapPixelFormat.Gray8 == pixelFormat ? 1 : 3; List <Int64> shape = new List <Int64>() { 1, temp_len, channels, height, width }; // B,T,C,H,W // The channels of image stored in buffer is in order of BGRA-BGRA-BGRA-BGRA. // Then we transform it to the order of BBBBB....GGGGG....RRRR....AAAA(dropped) TensorFloat tf = TensorFloat.Create(shape); byte * pCPUTensorbyte; float * pCPUTensor; uint uCapacity; // The channels of image stored in buffer is in order of BGRA-BGRA-BGRA-BGRA. // Then we transform it to the order of BBBBB....GGGGG....RRRR....AAAA(dropped) var tfr = tf.CreateReference(); var tfr2 = (IMemoryBufferByteAccess)tfr; tfr2.GetBuffer(out pCPUTensorbyte, out uCapacity); pCPUTensor = (float *)pCPUTensorbyte; for (Int32 t = 0; t < temp_len; t += 1) { VideoFrame frame = frameList[t]; SoftwareBitmap softwareBitmap2 = frame.SoftwareBitmap; // 1. Get the access to buffer of softwarebitmap BitmapBuffer spBitmapBuffer = softwareBitmap2.LockBuffer(BitmapBufferAccessMode.Read); IMemoryBufferReference reference = spBitmapBuffer.CreateReference(); byte *pData; uint size; ((IMemoryBufferByteAccess)reference).GetBuffer(out pData, out size); // 2. Transform the data in buffer to a vector of float var offset = (height * width * channels) * t; if (BitmapPixelFormat.Bgra8 == pixelFormat) { for (UInt32 i = 0; i < size; i += 4) { if (toRGB) { // suppose the model expects BGR image. // index 0 is B, 1 is G, 2 is R, 3 is alpha(dropped). UInt32 pixelInd = i / 4; pCPUTensor[offset + (height * width * 0) + pixelInd] = (((float)pData[i + 2]) - mean[0]) / std[0]; pCPUTensor[offset + (height * width * 1) + pixelInd] = (((float)pData[i + 1]) - mean[1]) / std[1]; pCPUTensor[offset + (height * width * 2) + pixelInd] = (((float)pData[i + 0]) - mean[2]) / std[2]; } else { // suppose the model expects BGR image. // index 0 is B, 1 is G, 2 is R, 3 is alpha(dropped). UInt32 pixelInd = i / 4; pCPUTensor[offset + (height * width * 0) + pixelInd] = (((float)pData[i + 0]) - mean[0]) / std[0]; pCPUTensor[offset + (height * width * 1) + pixelInd] = (((float)pData[i + 1]) - mean[1]) / std[1]; pCPUTensor[offset + (height * width * 2) + pixelInd] = (((float)pData[i + 2]) - mean[2]) / std[2]; } } } else if (BitmapPixelFormat.Rgba8 == pixelFormat) { for (UInt32 i = 0; i < size; i += 4) { // suppose the model expects BGR image. // index 0 is B, 1 is G, 2 is R, 3 is alpha(dropped). if (toRGB) { // suppose the model expects BGR image. // index 0 is B, 1 is G, 2 is R, 3 is alpha(dropped). UInt32 pixelInd = i / 4; pCPUTensor[offset + (height * width * 0) + pixelInd] = (((float)pData[i + 0]) - mean[0]) / std[0]; pCPUTensor[offset + (height * width * 1) + pixelInd] = (((float)pData[i + 1]) - mean[1]) / std[1]; pCPUTensor[offset + (height * width * 2) + pixelInd] = (((float)pData[i + 2]) - mean[2]) / std[2]; } else { UInt32 pixelInd = i / 4; pCPUTensor[offset + (height * width * 0) + pixelInd] = (((float)pData[i + 2]) - mean[0]) / std[0]; pCPUTensor[offset + (height * width * 1) + pixelInd] = (((float)pData[i + 1]) - mean[1]) / std[1]; pCPUTensor[offset + (height * width * 2) + pixelInd] = (((float)pData[i + 0]) - mean[2]) / std[2]; } } } else if (BitmapPixelFormat.Gray8 == pixelFormat) { for (UInt32 i = 0; i < size; i += 4) { // suppose the model expects BGR image. // index 0 is B, 1 is G, 2 is R, 3 is alpha(dropped). UInt32 pixelInd = i / 4; float red = (float)pData[i + 2]; float green = (float)pData[i + 1]; float blue = (float)pData[i]; float gray = 0.2126f * red + 0.7152f * green + 0.0722f * blue; pCPUTensor[offset + pixelInd] = gray; } } } // to prepend following error, copy to another instance and use it as model input. // The tensor has outstanding memory buffer references that must be closed prior to evaluation! TensorFloat ret = TensorFloat.CreateFromIterable( tf.Shape, tf.GetAsVectorView()); return(ret); }
#pragma warning disable CA1416 // Validate platform compatibility private void ApplyEffects(bool recreateSession = true) { if (!initialized_ || decoder_ == null) { return; } PerformanceMetricsMonitor.ClearLog(); if (recreateSession) { RecreateSessions(); } // TensorizeWithVideoFrame(); long start, stop; // Tensorize start = HighResolutionClock.UtcNow(); var pixelDataProvider = decoder_.GetPixelDataAsync().GetAwaiter().GetResult(); var bytes = pixelDataProvider.DetachPixelData(); var buffer = bytes.AsBuffer(); // Does this do a copy?? var inputRawTensor = TensorUInt8Bit.CreateFromBuffer(new long[] { 1, buffer.Length }, buffer); // 3 channel NCHW var nextOutputShape = new long[] { 1, 3, currentImageHeight_, currentImageWidth_ }; var intermediateTensor = TensorFloat.Create(nextOutputShape); var tensorizationBinding = Evaluate(tensorizationSession_, inputRawTensor, intermediateTensor); stop = HighResolutionClock.UtcNow(); var tensorizationDuration = HighResolutionClock.DurationInMs(start, stop); // Resize start = HighResolutionClock.UtcNow(); TensorFloat resizeOutput = null; LearningModelBinding resizeBinding = null; if (resizeEffectSession_ != null) { nextOutputShape = new long[] { 1, 3, 224, 224 }; resizeOutput = TensorFloat.Create(nextOutputShape); resizeBinding = Evaluate(resizeEffectSession_, intermediateTensor, resizeOutput); intermediateTensor = resizeOutput; } stop = HighResolutionClock.UtcNow(); var resizeDuration = HighResolutionClock.DurationInMs(start, stop); // Pixel Swizzle start = HighResolutionClock.UtcNow(); TensorFloat swizzleOutput = null; LearningModelBinding swizzleBinding = null; if (pixelSwizzleEffectSession_ != null) { swizzleOutput = TensorFloat.Create(nextOutputShape); swizzleBinding = Evaluate(pixelSwizzleEffectSession_, intermediateTensor, swizzleOutput); intermediateTensor = swizzleOutput; } stop = HighResolutionClock.UtcNow(); var swizzleDuration = HighResolutionClock.DurationInMs(start, stop); // Blur start = HighResolutionClock.UtcNow(); TensorFloat blurOutput = null; LearningModelBinding blurBinding = null; if (blurSharpenEffectSession_ != null) { blurOutput = TensorFloat.Create(nextOutputShape); blurBinding = Evaluate(blurSharpenEffectSession_, intermediateTensor, blurOutput); intermediateTensor = blurOutput; } stop = HighResolutionClock.UtcNow(); var blurDuration = HighResolutionClock.DurationInMs(start, stop); // Contrast start = HighResolutionClock.UtcNow(); TensorFloat contrastOutput = null; LearningModelBinding contrastBinding = null; if (contrastEffectSession_ != null) { contrastOutput = TensorFloat.Create(nextOutputShape); contrastBinding = EvaluateContrastAndBrightnessSession(intermediateTensor, contrastOutput); intermediateTensor = contrastOutput; } stop = HighResolutionClock.UtcNow(); var contrastDuration = HighResolutionClock.DurationInMs(start, stop); // Artistic Effects start = HighResolutionClock.UtcNow(); LearningModelBinding artistiicEffectsBinding = null; if (artisticEffectsEffectSession_ != null) { var output = TensorFloat.Create(nextOutputShape); artistiicEffectsBinding = Evaluate(artisticEffectsEffectSession_, intermediateTensor, output); intermediateTensor = output; } stop = HighResolutionClock.UtcNow(); var artisticEffectsDuration = HighResolutionClock.DurationInMs(start, stop); // Orientation start = HighResolutionClock.UtcNow(); TensorFloat orientationOutput = null; LearningModelBinding orientationBinding = null; if (orientationEffectSession_ != null) { var orientationEffect = (OrientationPicker.SelectedItem as OrientationViewModel).Tag; if (orientationEffect == Effect.RotateLeft90 || orientationEffect == Effect.RotateRight90) { nextOutputShape = new long[] { 1, 3, nextOutputShape[3], nextOutputShape[2] }; orientationOutput = TensorFloat.Create(nextOutputShape); } else { orientationOutput = TensorFloat.Create(nextOutputShape); } orientationBinding = Evaluate(orientationEffectSession_, intermediateTensor, orientationOutput); intermediateTensor = orientationOutput; } stop = HighResolutionClock.UtcNow(); var orientationDuration = HighResolutionClock.DurationInMs(start, stop); // Detensorize start = HighResolutionClock.UtcNow(); var shape = intermediateTensor.Shape; var n = (int)shape[0]; var c = (int)shape[1]; var h = (int)shape[2]; var w = (int)shape[3]; // Rather than writing the data into the software bitmap ourselves from a Tensor (which may be on the gpu) // we call an indentity model to move the gpu memory back to the cpu via WinML de-tensorization. var outputImage = new SoftwareBitmap(BitmapPixelFormat.Bgra8, w, h, BitmapAlphaMode.Premultiplied); var outputFrame = VideoFrame.CreateWithSoftwareBitmap(outputImage); var descriptor = detensorizationSession_.Model.InputFeatures[0] as TensorFeatureDescriptor; var detensorizerShape = descriptor.Shape; if (c != detensorizerShape[1] || h != detensorizerShape[2] || w != detensorizerShape[3]) { detensorizationSession_ = CreateLearningModelSession(TensorizationModels.IdentityNCHW(n, c, h, w)); } var detensorizationBinding = Evaluate(detensorizationSession_, intermediateTensor, outputFrame, true); stop = HighResolutionClock.UtcNow(); var detensorizationDuration = HighResolutionClock.DurationInMs(start, stop); // Render var softwareBitmap = outputFrame.SoftwareBitmap; RenderingHelpers.BindSoftwareBitmapToImageControl(InputImage, softwareBitmap); PerformanceMetricsMonitor.Log("Tensorize", tensorizationDuration); PerformanceMetricsMonitor.Log("Resize Effect", resizeDuration); PerformanceMetricsMonitor.Log("Swizzle Effect", swizzleDuration); PerformanceMetricsMonitor.Log("Blur Effect", blurDuration); PerformanceMetricsMonitor.Log("Contrast Effect", contrastDuration); PerformanceMetricsMonitor.Log("Artistic Effect", artisticEffectsDuration); PerformanceMetricsMonitor.Log("Orientation Effect", orientationDuration); PerformanceMetricsMonitor.Log("Detensorize", detensorizationDuration); }