private void SampleInputsGridView_SelectionChanged(object sender, SelectionChangedEventArgs e) { var gridView = sender as GridView; var thumbnail = gridView.SelectedItem as WinMLSamplesGallery.Controls.Thumbnail; if (thumbnail != null) { var image = thumbnail.ImageUri; var file = StorageFile.GetFileFromApplicationUriAsync(new Uri(image)).GetAwaiter().GetResult(); var softwareBitmap = CreateSoftwareBitmapFromStorageFile(file); tensorizationSession_ = CreateLearningModelSession( TensorizationModels.ReshapeFlatBufferNHWC( 1, 4, softwareBitmap.PixelHeight, softwareBitmap.PixelWidth, 416, 416)); // Tensorize var stream = file.OpenAsync(FileAccessMode.Read).GetAwaiter().GetResult(); var decoder = BitmapDecoder.CreateAsync(stream).GetAwaiter().GetResult(); var bitmap = decoder.GetSoftwareBitmapAsync(BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied).GetAwaiter().GetResult(); var pixelDataProvider = decoder.GetPixelDataAsync().GetAwaiter().GetResult(); var bytes = pixelDataProvider.DetachPixelData(); var buffer = bytes.AsBuffer(); // Does this do a copy?? var inputRawTensor = TensorUInt8Bit.CreateFromBuffer(new long[] { 1, buffer.Length }, buffer); // 3 channel NCHW var tensorizeOutput = TensorFloat.Create(new long[] { 1, 416, 416, 3 }); var b = new LearningModelBinding(tensorizationSession_); b.Bind(tensorizationSession_.Model.InputFeatures[0].Name, inputRawTensor); b.Bind(tensorizationSession_.Model.OutputFeatures[0].Name, tensorizeOutput); tensorizationSession_.Evaluate(b, ""); // Resize var resizeBinding = new LearningModelBinding(_session); resizeBinding.Bind(_session.Model.InputFeatures[0].Name, tensorizeOutput); var results = _session.Evaluate(resizeBinding, ""); var output1 = results.Output(0) as TensorFloat; var data = output1.GetAsVectorView(); var detections = ParseResult(data.ToList <float>().ToArray()); Comparer cp = new Comparer(); detections.Sort(cp); var final = NMS(detections); RenderImageInMainPanel(softwareBitmap); } }
private void DecryptAndEvauluate() { // Load the encrypted model. // The encrypted model (encrypted.onnx) is embedded as a resource in // the native binary: WinMLSamplesGalleryNative.dll. var inferenceModel = WinMLSamplesGalleryNative.EncryptedModels.LoadEncryptedResource(DecryptionKey.Password); var postProcessingModel = TensorizationModels.SoftMaxThenTopK(10); // Update the status var isModelDecrypted = inferenceModel != null; UpdateStatus(isModelDecrypted); // If loading the decrypted model failed (ie: due to an invalid key/password), // then skip performing evaluate. if (!isModelDecrypted) { return; } // Draw the image to classify in the Image control var decoder = ImageHelper.CreateBitmapDecoderFromPath("ms-appx:///InputData/hummingbird.jpg"); // Create sessions var device = new LearningModelDevice(LearningModelDeviceKind.Cpu); var options = new LearningModelSessionOptions() { CloseModelOnSessionCreation = true // Close the model to prevent extra memory usage }; var inferenceSession = new LearningModelSession(inferenceModel, device, options); var postProcessingSession = new LearningModelSession(postProcessingModel, device, options); // Classify the current image var softwareBitmap = decoder.GetSoftwareBitmapAsync().GetAwaiter().GetResult(); var input = VideoFrame.CreateWithSoftwareBitmap(softwareBitmap); // Inference var inferenceResults = Evaluate(inferenceSession, input); var inferenceOutput = inferenceResults.Outputs.First().Value; // PostProcess var postProcessedOutputs = Evaluate(postProcessingSession, inferenceOutput); var topKValues = (TensorFloat)postProcessedOutputs.Outputs["TopKValues"]; var topKIndices = (TensorInt64Bit)postProcessedOutputs.Outputs["TopKIndices"]; // Return results var probabilities = topKValues.GetAsVectorView(); var indices = topKIndices.GetAsVectorView(); var labels = indices.Select((index) => ClassificationLabels.ImageNet[index]); // Render the classification and probabilities RenderInferenceResults(labels, probabilities); }
private void RecreateSessions() { tensorizationSession_?.Dispose(); tensorizationSession_ = CreateLearningModelSession( TensorizationModels.ReshapeFlatBufferToNCHW( 1, 4, currentImageHeight_, currentImageWidth_)); resizeEffectSession_?.Dispose(); resizeEffectSession_ = GetEffect(ResizeToggleSplitButton, ResizePicker); pixelSwizzleEffectSession_?.Dispose(); pixelSwizzleEffectSession_ = GetPixelSwizzleEffect(); blurSharpenEffectSession_?.Dispose(); blurSharpenEffectSession_ = GetEffect(BlurSharpenToggleSplitButton, BlurSharpenPicker); contrastEffectSession_?.Dispose(); contrastEffectSession_ = ContrastToggleSplitButton.IsChecked ? CreateLearningModelSession(TensorizationModels.AdjustBrightnessAndContrast( 1, 3, resizeEffectSession_ != null ? 224 : currentImageHeight_, resizeEffectSession_ != null ? 224 : currentImageWidth_)) : null; artisticEffectsEffectSession_?.Dispose(); artisticEffectsEffectSession_ = GetEffect(ArtisticEffectsToggleSplitButton, ArtisticEffectsPicker); orientationEffectSession_?.Dispose(); orientationEffectSession_ = GetOrientationEffect(); shapeSession_?.Dispose(); shapeSession_ = CreateLearningModelSession(TensorizationModels.ShapeNCHW(1, 3, currentImageHeight_, currentImageWidth_)); detensorizationSession_?.Dispose(); detensorizationSession_ = CreateLearningModelSession(TensorizationModels.IdentityNCHW( 1, 3, resizeEffectSession_ != null ? 224 : currentImageHeight_, resizeEffectSession_ != null ? 224 : currentImageWidth_)); }
public OpenCVInterop() { this.InitializeComponent(); CurrentImagePath = null; InferenceChoice = ClassifyChoice.Denoised; // Load inference session var modelName = "squeezenet1.1-7.onnx"; var modelPath = Path.Join(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, "Models", modelName); var model = LearningModel.LoadFromFilePath(modelPath); _inferenceSession = CreateLearningModelSession(model); // Load post processing session _postProcessingSession = CreateLearningModelSession(TensorizationModels.SoftMaxThenTopK(TopK)); BasicGridView.SelectedIndex = 0; }
private void TryPerformInference(bool reloadImages = true) { if (CurrentImagePath != null) { if (reloadImages) { Original = WinMLSamplesGalleryNative.OpenCVImage.CreateFromPath(CurrentImagePath); Noisy = WinMLSamplesGalleryNative.OpenCVImage.AddSaltAndPepperNoise(Original); Denoised = WinMLSamplesGalleryNative.OpenCVImage.DenoiseMedianBlur(Noisy); var baseImageBitmap = Original.AsSoftwareBitmap(); RenderingHelpers.BindSoftwareBitmapToImageControl(InputImage, baseImageBitmap); RenderingHelpers.BindSoftwareBitmapToImageControl(NoisyImage, Noisy.AsSoftwareBitmap()); RenderingHelpers.BindSoftwareBitmapToImageControl(DenoisedImage, Denoised.AsSoftwareBitmap()); var tensorizationModel = TensorizationModels.CastResizeAndTranspose11(Height, Width, 1, 3, baseImageBitmap.PixelHeight, baseImageBitmap.PixelWidth, "nearest"); _tensorizationSession = CreateLearningModelSession(tensorizationModel, SelectedDeviceKind); } WinMLSamplesGalleryNative.OpenCVImage classificationImage = null; switch (InferenceChoice) { case ClassifyChoice.Original: classificationImage = Original; break; case ClassifyChoice.Noisy: classificationImage = Noisy; break; case ClassifyChoice.Denoised: classificationImage = Denoised; break; } // Classify var(labels, probabilities) = Classify(classificationImage); // Render the classification and probabilities RenderInferenceResults(labels, probabilities); } }
public ImageSharpInterop() { this.InitializeComponent(); var tensorizationModel = TensorizationModels.BasicTensorization(Height, Width, BatchSize, Channels, Height, Width, "nearest"); _tensorizationSession = CreateLearningModelSession(tensorizationModel, SelectedDeviceKind); var inferenceModelName = "squeezenet1.1-7.onnx"; var inferenceModelPath = Path.Join(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, "Models", inferenceModelName); var inferenceModel = LearningModel.LoadFromFilePath(inferenceModelPath); _inferenceSession = CreateLearningModelSession(inferenceModel); var postProcessingModel = TensorizationModels.SoftMaxThenTopK(TopK); _postProcessingSession = CreateLearningModelSession(postProcessingModel); BasicGridView.SelectedIndex = 0; }
private LearningModelSession GetEffectSession(Effect effect) { switch (effect) { case Effect.Blur3x3: return(CreateLearningModelSession(TensorizationModels.AveragePool(3))); case Effect.Blur5x5: return(CreateLearningModelSession(TensorizationModels.AveragePool(5))); case Effect.Blur7x7: return(CreateLearningModelSession(TensorizationModels.AveragePool(7))); case Effect.ResizeCubicFill: return(CreateLearningModelSession(TensorizationModels.UniformScaleAndCenterFill(currentImageHeight_, currentImageWidth_, 224, 224, "cubic"))); case Effect.ResizeCubicFit: return(CreateLearningModelSession(TensorizationModels.UniformScaleAndCenterFit(currentImageHeight_, currentImageWidth_, 224, 224, "cubic"))); case Effect.ResizeLinearFill: return(CreateLearningModelSession(TensorizationModels.UniformScaleAndCenterFill(currentImageHeight_, currentImageWidth_, 224, 224, "linear"))); case Effect.ResizeLinearFit: return(CreateLearningModelSession(TensorizationModels.UniformScaleAndCenterFit(currentImageHeight_, currentImageWidth_, 224, 224, "linear"))); case Effect.ResizeNearestFill: return(CreateLearningModelSession(TensorizationModels.UniformScaleAndCenterFill(currentImageHeight_, currentImageWidth_, 224, 224, "nearest"))); case Effect.ResizeNearestFit: return(CreateLearningModelSession(TensorizationModels.UniformScaleAndCenterFit(currentImageHeight_, currentImageWidth_, 224, 224, "nearest"))); case Effect.Sobel: return(CreateLearningModelSession(TensorizationModels.Sobel())); case Effect.PixelSwizzle_123_321: return(CreateLearningModelSession(TensorizationModels.Swizzle(3, 2, 1))); case Effect.PixelSwizzle_123_312: return(CreateLearningModelSession(TensorizationModels.Swizzle(3, 1, 2))); case Effect.PixelSwizzle_123_213: return(CreateLearningModelSession(TensorizationModels.Swizzle(2, 1, 3))); case Effect.PixelSwizzle_123_231: return(CreateLearningModelSession(TensorizationModels.Swizzle(2, 3, 1))); case Effect.PixelSwizzle_123_132: return(CreateLearningModelSession(TensorizationModels.Swizzle(1, 3, 2))); case Effect.MirrorHorizontal: return(CreateLearningModelSession(TensorizationModels.MirrorHorizontalNCHW())); case Effect.MirrorVertical: return(CreateLearningModelSession(TensorizationModels.MirrorVerticalNCHW())); case Effect.RotateRight90: return(CreateLearningModelSession(TensorizationModels.RotateRight90())); case Effect.RotateLeft90: return(CreateLearningModelSession(TensorizationModels.RotateLeft90())); default: return(null); } }
#pragma warning disable CA1416 // Validate platform compatibility private void ApplyEffects(bool recreateSession = true) { if (!initialized_ || decoder_ == null) { return; } PerformanceMetricsMonitor.ClearLog(); if (recreateSession) { RecreateSessions(); } // TensorizeWithVideoFrame(); long start, stop; // Tensorize start = HighResolutionClock.UtcNow(); var pixelDataProvider = decoder_.GetPixelDataAsync().GetAwaiter().GetResult(); var bytes = pixelDataProvider.DetachPixelData(); var buffer = bytes.AsBuffer(); // Does this do a copy?? var inputRawTensor = TensorUInt8Bit.CreateFromBuffer(new long[] { 1, buffer.Length }, buffer); // 3 channel NCHW var nextOutputShape = new long[] { 1, 3, currentImageHeight_, currentImageWidth_ }; var intermediateTensor = TensorFloat.Create(nextOutputShape); var tensorizationBinding = Evaluate(tensorizationSession_, inputRawTensor, intermediateTensor); stop = HighResolutionClock.UtcNow(); var tensorizationDuration = HighResolutionClock.DurationInMs(start, stop); // Resize start = HighResolutionClock.UtcNow(); TensorFloat resizeOutput = null; LearningModelBinding resizeBinding = null; if (resizeEffectSession_ != null) { nextOutputShape = new long[] { 1, 3, 224, 224 }; resizeOutput = TensorFloat.Create(nextOutputShape); resizeBinding = Evaluate(resizeEffectSession_, intermediateTensor, resizeOutput); intermediateTensor = resizeOutput; } stop = HighResolutionClock.UtcNow(); var resizeDuration = HighResolutionClock.DurationInMs(start, stop); // Pixel Swizzle start = HighResolutionClock.UtcNow(); TensorFloat swizzleOutput = null; LearningModelBinding swizzleBinding = null; if (pixelSwizzleEffectSession_ != null) { swizzleOutput = TensorFloat.Create(nextOutputShape); swizzleBinding = Evaluate(pixelSwizzleEffectSession_, intermediateTensor, swizzleOutput); intermediateTensor = swizzleOutput; } stop = HighResolutionClock.UtcNow(); var swizzleDuration = HighResolutionClock.DurationInMs(start, stop); // Blur start = HighResolutionClock.UtcNow(); TensorFloat blurOutput = null; LearningModelBinding blurBinding = null; if (blurSharpenEffectSession_ != null) { blurOutput = TensorFloat.Create(nextOutputShape); blurBinding = Evaluate(blurSharpenEffectSession_, intermediateTensor, blurOutput); intermediateTensor = blurOutput; } stop = HighResolutionClock.UtcNow(); var blurDuration = HighResolutionClock.DurationInMs(start, stop); // Contrast start = HighResolutionClock.UtcNow(); TensorFloat contrastOutput = null; LearningModelBinding contrastBinding = null; if (contrastEffectSession_ != null) { contrastOutput = TensorFloat.Create(nextOutputShape); contrastBinding = EvaluateContrastAndBrightnessSession(intermediateTensor, contrastOutput); intermediateTensor = contrastOutput; } stop = HighResolutionClock.UtcNow(); var contrastDuration = HighResolutionClock.DurationInMs(start, stop); // Artistic Effects start = HighResolutionClock.UtcNow(); LearningModelBinding artistiicEffectsBinding = null; if (artisticEffectsEffectSession_ != null) { var output = TensorFloat.Create(nextOutputShape); artistiicEffectsBinding = Evaluate(artisticEffectsEffectSession_, intermediateTensor, output); intermediateTensor = output; } stop = HighResolutionClock.UtcNow(); var artisticEffectsDuration = HighResolutionClock.DurationInMs(start, stop); // Orientation start = HighResolutionClock.UtcNow(); TensorFloat orientationOutput = null; LearningModelBinding orientationBinding = null; if (orientationEffectSession_ != null) { var orientationEffect = (OrientationPicker.SelectedItem as OrientationViewModel).Tag; if (orientationEffect == Effect.RotateLeft90 || orientationEffect == Effect.RotateRight90) { nextOutputShape = new long[] { 1, 3, nextOutputShape[3], nextOutputShape[2] }; orientationOutput = TensorFloat.Create(nextOutputShape); } else { orientationOutput = TensorFloat.Create(nextOutputShape); } orientationBinding = Evaluate(orientationEffectSession_, intermediateTensor, orientationOutput); intermediateTensor = orientationOutput; } stop = HighResolutionClock.UtcNow(); var orientationDuration = HighResolutionClock.DurationInMs(start, stop); // Detensorize start = HighResolutionClock.UtcNow(); var shape = intermediateTensor.Shape; var n = (int)shape[0]; var c = (int)shape[1]; var h = (int)shape[2]; var w = (int)shape[3]; // Rather than writing the data into the software bitmap ourselves from a Tensor (which may be on the gpu) // we call an indentity model to move the gpu memory back to the cpu via WinML de-tensorization. var outputImage = new SoftwareBitmap(BitmapPixelFormat.Bgra8, w, h, BitmapAlphaMode.Premultiplied); var outputFrame = VideoFrame.CreateWithSoftwareBitmap(outputImage); var descriptor = detensorizationSession_.Model.InputFeatures[0] as TensorFeatureDescriptor; var detensorizerShape = descriptor.Shape; if (c != detensorizerShape[1] || h != detensorizerShape[2] || w != detensorizerShape[3]) { detensorizationSession_ = CreateLearningModelSession(TensorizationModels.IdentityNCHW(n, c, h, w)); } var detensorizationBinding = Evaluate(detensorizationSession_, intermediateTensor, outputFrame, true); stop = HighResolutionClock.UtcNow(); var detensorizationDuration = HighResolutionClock.DurationInMs(start, stop); // Render var softwareBitmap = outputFrame.SoftwareBitmap; RenderingHelpers.BindSoftwareBitmapToImageControl(InputImage, softwareBitmap); PerformanceMetricsMonitor.Log("Tensorize", tensorizationDuration); PerformanceMetricsMonitor.Log("Resize Effect", resizeDuration); PerformanceMetricsMonitor.Log("Swizzle Effect", swizzleDuration); PerformanceMetricsMonitor.Log("Blur Effect", blurDuration); PerformanceMetricsMonitor.Log("Contrast Effect", contrastDuration); PerformanceMetricsMonitor.Log("Artistic Effect", artisticEffectsDuration); PerformanceMetricsMonitor.Log("Orientation Effect", orientationDuration); PerformanceMetricsMonitor.Log("Detensorize", detensorizationDuration); }