private (IEnumerable <string>, IReadOnlyList <float>) Classify(Image <Bgra32> image, float angle)
        {
            long start, stop;

            PerformanceMetricsMonitor.ClearLog();

            // Tensorize
            start = HighResolutionClock.UtcNow();
            image.Mutate(ctx => ctx.Rotate(angle));
            var resizeOptions = new ResizeOptions()
            {
                Mode = ResizeMode.Crop,
                Size = new SixLabors.ImageSharp.Size((int)Width, (int)Height)
            };

            image.Mutate(ctx => ctx.Resize(resizeOptions));
            object input = image.AsTensor();
            var    tensorizationResults = Evaluate(_tensorizationSession, input);
            object tensorizedOutput     = tensorizationResults.Outputs.First().Value;

            stop = HighResolutionClock.UtcNow();
            var tensorizeDuration = HighResolutionClock.DurationInMs(start, stop);

            // Inference
            start = HighResolutionClock.UtcNow();
            var inferenceResults = Evaluate(_inferenceSession, tensorizedOutput);
            var inferenceOutput  = inferenceResults.Outputs.First().Value;

            stop = HighResolutionClock.UtcNow();
            var inferenceDuration = HighResolutionClock.DurationInMs(start, stop);

            // PostProcess
            start = HighResolutionClock.UtcNow();
            var postProcessedOutputs = Evaluate(_postProcessingSession, inferenceOutput);
            var topKValues           = (TensorFloat)postProcessedOutputs.Outputs["TopKValues"];
            var topKIndices          = (TensorInt64Bit)postProcessedOutputs.Outputs["TopKIndices"];

            // Return results
            var probabilities = topKValues.GetAsVectorView();
            var indices       = topKIndices.GetAsVectorView();
            var labels        = indices.Select((index) => ClassificationLabels.ImageNet[index]);

            stop = HighResolutionClock.UtcNow();
            var postProcessDuration = HighResolutionClock.DurationInMs(start, stop);

            PerformanceMetricsMonitor.Log("Tensorize", tensorizeDuration);
            PerformanceMetricsMonitor.Log("Pre-process", 0);
            PerformanceMetricsMonitor.Log("Inference", inferenceDuration);
            PerformanceMetricsMonitor.Log("Post-process", postProcessDuration);

            RenderingHelpers.BindSoftwareBitmapToImageControl(InputImage, image.AsSoftwareBitmap());
            image.Dispose();

            return(labels, probabilities);
        }
        private void SetCurrentImage(string path)
        {
            if (CurrentImage != null)
            {
                CurrentImage.Dispose();
            }

            RotationSlider.IsEnabled = true;

            CurrentImage = SixLabors.ImageSharp.Image.Load <Bgra32>(path);
            RenderingHelpers.BindSoftwareBitmapToImageControl(InputImage, CurrentImage.AsSoftwareBitmap());
        }
        private void TryPerformInference(bool reloadImages = true)
        {
            if (CurrentImagePath != null)
            {
                if (reloadImages)
                {
                    Original = WinMLSamplesGalleryNative.OpenCVImage.CreateFromPath(CurrentImagePath);
                    Noisy    = WinMLSamplesGalleryNative.OpenCVImage.AddSaltAndPepperNoise(Original);
                    Denoised = WinMLSamplesGalleryNative.OpenCVImage.DenoiseMedianBlur(Noisy);

                    var baseImageBitmap = Original.AsSoftwareBitmap();
                    RenderingHelpers.BindSoftwareBitmapToImageControl(InputImage, baseImageBitmap);
                    RenderingHelpers.BindSoftwareBitmapToImageControl(NoisyImage, Noisy.AsSoftwareBitmap());
                    RenderingHelpers.BindSoftwareBitmapToImageControl(DenoisedImage, Denoised.AsSoftwareBitmap());

                    var tensorizationModel = TensorizationModels.CastResizeAndTranspose11(Height, Width, 1, 3, baseImageBitmap.PixelHeight, baseImageBitmap.PixelWidth, "nearest");
                    _tensorizationSession = CreateLearningModelSession(tensorizationModel, SelectedDeviceKind);
                }

                WinMLSamplesGalleryNative.OpenCVImage classificationImage = null;
                switch (InferenceChoice)
                {
                case ClassifyChoice.Original:
                    classificationImage = Original;
                    break;

                case ClassifyChoice.Noisy:
                    classificationImage = Noisy;
                    break;

                case ClassifyChoice.Denoised:
                    classificationImage = Denoised;
                    break;
                }

                // Classify
                var(labels, probabilities) = Classify(classificationImage);

                // Render the classification and probabilities
                RenderInferenceResults(labels, probabilities);
            }
        }
示例#4
0
        public HtmlString Render()
        {
            var renderHelp = new RenderingHelpers(this);

            try
            {
                string innerContent = null;

                #region do pre-check to see if system is stable & ready
                var notReady = new InstallationController().CheckUpgradeMessage(PortalSettings.Current.UserInfo.IsSuperUser);
                if (!string.IsNullOrEmpty(notReady))
                {
                    innerContent = renderHelp.DesignErrorMessage(new Exception(notReady), true, "Error - needs admin to fix this", false, false);
                }

                #endregion

                #region check if the content-group exists (sometimes it's missing if a site is being imported and the data isn't in yet

                if (ContentBlock.DataIsMissing)
                {
                    if (Environment.Permissions.UserMayEditContent)
                    {
                        innerContent = ""; // stop further processing
                    }
                    else // end users should see server error as no js-side processing will happen
                    {
                        var ex =
                            new Exception(
                                "Data is missing - usually when a site is copied but the content / apps have not been imported yet - check 2sxc.org/help?tag=export-import");
                        innerContent = renderHelp.DesignErrorMessage(ex, true,
                                                                     "Error - needs admin to fix", false, true);
                    }
                }
                #endregion

                #region try to render the block or generate the error message
                if (innerContent == null)
                {
                    try
                    {
                        if (Template != null) // when a content block is still new, there is no definition yet
                        {
                            var engine = GetRenderingEngine(InstancePurposes.WebView);
                            innerContent = engine.Render();
                        }
                        else
                        {
                            innerContent = "";
                        }
                    }
                    catch (Exception ex)
                    {
                        innerContent = renderHelp.DesignErrorMessage(ex, true, "Error rendering template", false, true);
                    }
                }
                #endregion

                #region Wrap it all up into a nice wrapper tag
                var editInfos = JsonConvert.SerializeObject(renderHelp.GetClientInfosAll());
                var startTag  = (RenderWithDiv
                    ? $"<div class=\"sc-viewport sc-content-block\" data-cb-instance=\"{ContentBlock.ParentId}\" " +
                                 $" data-cb-id=\"{ContentBlock.ContentBlockId}\""
                                 +
                                 (RenderWithEditMetadata
                          ? " data-edit-context=\'" + editInfos + "'"
                          : "")
                                 + ">\n"
                    : "");
                var    endTag = (RenderWithDiv ? "\n</div>" : "");
                string result = startTag + innerContent + endTag;
                #endregion

                return(new HtmlString(result));
            }
            catch (Exception ex)
            {
                // todo: i18n
                return(new HtmlString(renderHelp.DesignErrorMessage(ex, true, null, true, true)));
            }
        }
示例#5
0
#pragma warning disable CA1416 // Validate platform compatibility
        private void ApplyEffects(bool recreateSession = true)
        {
            if (!initialized_ || decoder_ == null)
            {
                return;
            }

            PerformanceMetricsMonitor.ClearLog();

            if (recreateSession)
            {
                RecreateSessions();
            }

            // TensorizeWithVideoFrame();

            long start, stop;

            // Tensorize
            start = HighResolutionClock.UtcNow();
            var pixelDataProvider = decoder_.GetPixelDataAsync().GetAwaiter().GetResult();
            var bytes             = pixelDataProvider.DetachPixelData();
            var buffer            = bytes.AsBuffer(); // Does this do a copy??
            var inputRawTensor    = TensorUInt8Bit.CreateFromBuffer(new long[] { 1, buffer.Length }, buffer);
            // 3 channel NCHW
            var nextOutputShape      = new long[] { 1, 3, currentImageHeight_, currentImageWidth_ };
            var intermediateTensor   = TensorFloat.Create(nextOutputShape);
            var tensorizationBinding = Evaluate(tensorizationSession_, inputRawTensor, intermediateTensor);

            stop = HighResolutionClock.UtcNow();
            var tensorizationDuration = HighResolutionClock.DurationInMs(start, stop);

            // Resize
            start = HighResolutionClock.UtcNow();
            TensorFloat          resizeOutput  = null;
            LearningModelBinding resizeBinding = null;

            if (resizeEffectSession_ != null)
            {
                nextOutputShape    = new long[] { 1, 3, 224, 224 };
                resizeOutput       = TensorFloat.Create(nextOutputShape);
                resizeBinding      = Evaluate(resizeEffectSession_, intermediateTensor, resizeOutput);
                intermediateTensor = resizeOutput;
            }
            stop = HighResolutionClock.UtcNow();
            var resizeDuration = HighResolutionClock.DurationInMs(start, stop);

            // Pixel Swizzle
            start = HighResolutionClock.UtcNow();
            TensorFloat          swizzleOutput  = null;
            LearningModelBinding swizzleBinding = null;

            if (pixelSwizzleEffectSession_ != null)
            {
                swizzleOutput      = TensorFloat.Create(nextOutputShape);
                swizzleBinding     = Evaluate(pixelSwizzleEffectSession_, intermediateTensor, swizzleOutput);
                intermediateTensor = swizzleOutput;
            }
            stop = HighResolutionClock.UtcNow();
            var swizzleDuration = HighResolutionClock.DurationInMs(start, stop);

            // Blur
            start = HighResolutionClock.UtcNow();
            TensorFloat          blurOutput  = null;
            LearningModelBinding blurBinding = null;

            if (blurSharpenEffectSession_ != null)
            {
                blurOutput         = TensorFloat.Create(nextOutputShape);
                blurBinding        = Evaluate(blurSharpenEffectSession_, intermediateTensor, blurOutput);
                intermediateTensor = blurOutput;
            }
            stop = HighResolutionClock.UtcNow();
            var blurDuration = HighResolutionClock.DurationInMs(start, stop);

            // Contrast
            start = HighResolutionClock.UtcNow();
            TensorFloat          contrastOutput  = null;
            LearningModelBinding contrastBinding = null;

            if (contrastEffectSession_ != null)
            {
                contrastOutput     = TensorFloat.Create(nextOutputShape);
                contrastBinding    = EvaluateContrastAndBrightnessSession(intermediateTensor, contrastOutput);
                intermediateTensor = contrastOutput;
            }
            stop = HighResolutionClock.UtcNow();
            var contrastDuration = HighResolutionClock.DurationInMs(start, stop);

            // Artistic Effects
            start = HighResolutionClock.UtcNow();
            LearningModelBinding artistiicEffectsBinding = null;

            if (artisticEffectsEffectSession_ != null)
            {
                var output = TensorFloat.Create(nextOutputShape);
                artistiicEffectsBinding = Evaluate(artisticEffectsEffectSession_, intermediateTensor, output);
                intermediateTensor      = output;
            }
            stop = HighResolutionClock.UtcNow();
            var artisticEffectsDuration = HighResolutionClock.DurationInMs(start, stop);

            // Orientation
            start = HighResolutionClock.UtcNow();
            TensorFloat          orientationOutput  = null;
            LearningModelBinding orientationBinding = null;

            if (orientationEffectSession_ != null)
            {
                var orientationEffect = (OrientationPicker.SelectedItem as OrientationViewModel).Tag;
                if (orientationEffect == Effect.RotateLeft90 ||
                    orientationEffect == Effect.RotateRight90)
                {
                    nextOutputShape   = new long[] { 1, 3, nextOutputShape[3], nextOutputShape[2] };
                    orientationOutput = TensorFloat.Create(nextOutputShape);
                }
                else
                {
                    orientationOutput = TensorFloat.Create(nextOutputShape);
                }
                orientationBinding = Evaluate(orientationEffectSession_, intermediateTensor, orientationOutput);
                intermediateTensor = orientationOutput;
            }
            stop = HighResolutionClock.UtcNow();
            var orientationDuration = HighResolutionClock.DurationInMs(start, stop);

            // Detensorize
            start = HighResolutionClock.UtcNow();
            var shape = intermediateTensor.Shape;
            var n     = (int)shape[0];
            var c     = (int)shape[1];
            var h     = (int)shape[2];
            var w     = (int)shape[3];

            // Rather than writing the data into the software bitmap ourselves from a Tensor (which may be on the gpu)
            // we call an indentity model to move the gpu memory back to the cpu via WinML de-tensorization.
            var outputImage = new SoftwareBitmap(BitmapPixelFormat.Bgra8, w, h, BitmapAlphaMode.Premultiplied);
            var outputFrame = VideoFrame.CreateWithSoftwareBitmap(outputImage);

            var descriptor        = detensorizationSession_.Model.InputFeatures[0] as TensorFeatureDescriptor;
            var detensorizerShape = descriptor.Shape;

            if (c != detensorizerShape[1] || h != detensorizerShape[2] || w != detensorizerShape[3])
            {
                detensorizationSession_ = CreateLearningModelSession(TensorizationModels.IdentityNCHW(n, c, h, w));
            }
            var detensorizationBinding = Evaluate(detensorizationSession_, intermediateTensor, outputFrame, true);

            stop = HighResolutionClock.UtcNow();
            var detensorizationDuration = HighResolutionClock.DurationInMs(start, stop);

            // Render
            var softwareBitmap = outputFrame.SoftwareBitmap;

            RenderingHelpers.BindSoftwareBitmapToImageControl(InputImage, softwareBitmap);

            PerformanceMetricsMonitor.Log("Tensorize", tensorizationDuration);
            PerformanceMetricsMonitor.Log("Resize Effect", resizeDuration);
            PerformanceMetricsMonitor.Log("Swizzle Effect", swizzleDuration);
            PerformanceMetricsMonitor.Log("Blur Effect", blurDuration);
            PerformanceMetricsMonitor.Log("Contrast Effect", contrastDuration);
            PerformanceMetricsMonitor.Log("Artistic Effect", artisticEffectsDuration);
            PerformanceMetricsMonitor.Log("Orientation Effect", orientationDuration);
            PerformanceMetricsMonitor.Log("Detensorize", detensorizationDuration);
        }
示例#6
0
        public HtmlString Render()
        {
            var renderHelp = new RenderingHelpers(this);

            try
            {
                string innerContent = null;

                #region do pre-check to see if system is stable & ready
                var notReady = new InstallationController().CheckUpgradeMessage(PortalSettings.Current.UserInfo.IsSuperUser);
                if (!string.IsNullOrEmpty(notReady))
                {
                    innerContent = renderHelp.DesignErrorMessage(new Exception(notReady), true, "Error - needs admin to fix this", false, false);
                }

                #endregion

                #region try to render the block or generate the error message
                if (innerContent == null)
                {
                    try
                    {
                        if (Template != null) // when a content block is still new, there is no definition yet
                        {
                            var engine = GetRenderingEngine(InstancePurposes.WebView);
                            innerContent = engine.Render();
                        }
                        else
                        {
                            innerContent = "";
                        }
                    }
                    catch (Exception ex)
                    {
                        innerContent = renderHelp.DesignErrorMessage(ex, true, "Error rendering template", false, true);
                    }
                }
                #endregion

                #region Wrap it all up into a nice wrapper tag
                var editInfos = JsonConvert.SerializeObject(renderHelp.GetClientInfosAll());
                var startTag  = (RenderWithDiv
                    ? $"<div class=\"sc-viewport sc-content-block\" data-cb-instance=\"{ContentBlock.ParentId}\" " +
                                 $" data-cb-id=\"{ContentBlock.ContentBlockId}\""
                                 +
                                 (RenderWithEditMetadata
                          ? " data-edit-context=\'" + editInfos + "'"
                          : "")
                                 + ">\n"
                    : "");
                var    endTag = (RenderWithDiv ? "\n</div>" : "");
                string result = startTag + innerContent + endTag;
                #endregion

                return(new HtmlString(result));
            }
            catch (Exception ex)
            {
                // todo: i18n
                return(new HtmlString(renderHelp.DesignErrorMessage(ex, true, null, true, true)));
            }
        }