private (IEnumerable <string>, IReadOnlyList <float>) Classify(Image <Bgra32> image, float angle)
        {
            long start, stop;

            PerformanceMetricsMonitor.ClearLog();

            // Tensorize
            start = HighResolutionClock.UtcNow();
            image.Mutate(ctx => ctx.Rotate(angle));
            var resizeOptions = new ResizeOptions()
            {
                Mode = ResizeMode.Crop,
                Size = new SixLabors.ImageSharp.Size((int)Width, (int)Height)
            };

            image.Mutate(ctx => ctx.Resize(resizeOptions));
            object input = image.AsTensor();
            var    tensorizationResults = Evaluate(_tensorizationSession, input);
            object tensorizedOutput     = tensorizationResults.Outputs.First().Value;

            stop = HighResolutionClock.UtcNow();
            var tensorizeDuration = HighResolutionClock.DurationInMs(start, stop);

            // Inference
            start = HighResolutionClock.UtcNow();
            var inferenceResults = Evaluate(_inferenceSession, tensorizedOutput);
            var inferenceOutput  = inferenceResults.Outputs.First().Value;

            stop = HighResolutionClock.UtcNow();
            var inferenceDuration = HighResolutionClock.DurationInMs(start, stop);

            // PostProcess
            start = HighResolutionClock.UtcNow();
            var postProcessedOutputs = Evaluate(_postProcessingSession, inferenceOutput);
            var topKValues           = (TensorFloat)postProcessedOutputs.Outputs["TopKValues"];
            var topKIndices          = (TensorInt64Bit)postProcessedOutputs.Outputs["TopKIndices"];

            // Return results
            var probabilities = topKValues.GetAsVectorView();
            var indices       = topKIndices.GetAsVectorView();
            var labels        = indices.Select((index) => ClassificationLabels.ImageNet[index]);

            stop = HighResolutionClock.UtcNow();
            var postProcessDuration = HighResolutionClock.DurationInMs(start, stop);

            PerformanceMetricsMonitor.Log("Tensorize", tensorizeDuration);
            PerformanceMetricsMonitor.Log("Pre-process", 0);
            PerformanceMetricsMonitor.Log("Inference", inferenceDuration);
            PerformanceMetricsMonitor.Log("Post-process", postProcessDuration);

            RenderingHelpers.BindSoftwareBitmapToImageControl(InputImage, image.AsSoftwareBitmap());
            image.Dispose();

            return(labels, probabilities);
        }
        private (IEnumerable <string>, IReadOnlyList <float>) Classify(WinMLSamplesGalleryNative.OpenCVImage image)
        {
            long start, stop;

            PerformanceMetricsMonitor.ClearLog();

            start = HighResolutionClock.UtcNow();
            object input = image.AsTensor();

            stop = HighResolutionClock.UtcNow();
            var pixelAccessDuration = HighResolutionClock.DurationInMs(start, stop);

            // Tensorize
            start = HighResolutionClock.UtcNow();
            object tensorizedOutput     = input;
            var    tensorizationResults = Evaluate(_tensorizationSession, input);

            tensorizedOutput = tensorizationResults.Outputs.First().Value;
            stop             = HighResolutionClock.UtcNow();
            var tensorizeDuration = HighResolutionClock.DurationInMs(start, stop);

            // Inference
            start = HighResolutionClock.UtcNow();
            var inferenceResults = Evaluate(_inferenceSession, tensorizedOutput);
            var inferenceOutput  = inferenceResults.Outputs.First().Value;

            stop = HighResolutionClock.UtcNow();
            var inferenceDuration = HighResolutionClock.DurationInMs(start, stop);

            // PostProcess
            start = HighResolutionClock.UtcNow();
            var postProcessedOutputs = Evaluate(_postProcessingSession, inferenceOutput);
            var topKValues           = (TensorFloat)postProcessedOutputs.Outputs["TopKValues"];
            var topKIndices          = (TensorInt64Bit)postProcessedOutputs.Outputs["TopKIndices"];

            // Return results
            var probabilities = topKValues.GetAsVectorView();
            var indices       = topKIndices.GetAsVectorView();
            var labels        = indices.Select((index) => ClassificationLabels.ImageNet[index]);

            stop = HighResolutionClock.UtcNow();
            var postProcessDuration = HighResolutionClock.DurationInMs(start, stop);

            PerformanceMetricsMonitor.Log("Pixel Access (CPU)", pixelAccessDuration);
            PerformanceMetricsMonitor.Log("Tensorize", tensorizeDuration);
            PerformanceMetricsMonitor.Log("Pre-process", 0);
            PerformanceMetricsMonitor.Log("Inference", inferenceDuration);
            PerformanceMetricsMonitor.Log("Post-process", postProcessDuration);

            return(labels, probabilities);
        }
Exemple #3
0
        private static float Evaluate(LearningModelSession session, List <VideoFrame> input)
        {
            string inputName     = session.Model.InputFeatures[0].Name;
            float  totalDuration = 0;
            var    binding       = new LearningModelBinding(session);

            for (int j = 0; j < input.Count; j++)
            {
                if (navigatingAwayFromPage)
                {
                    break;
                }

                var start = HighResolutionClock.UtcNow();
                binding.Bind(inputName, input[j]);
                session.Evaluate(binding, "");
                var stop     = HighResolutionClock.UtcNow();
                var duration = HighResolutionClock.DurationInMs(start, stop);
                totalDuration += duration;
            }
            return(totalDuration);
        }
Exemple #4
0
        private static float EvaluateBatched(LearningModelSession session, List <VideoFrame> input, int batchSize)
        {
            int    numBatches    = (int)Math.Ceiling((Decimal)input.Count / batchSize);
            string inputName     = session.Model.InputFeatures[0].Name;
            float  totalDuration = 0;
            var    binding       = new LearningModelBinding(session);

            for (int i = 0; i < numBatches; i++)
            {
                if (navigatingAwayFromPage)
                {
                    break;
                }

                int rangeStart = batchSize * i;
                List <VideoFrame> batch;
                // Add padding to the last batch if necessary
                if (rangeStart + batchSize > input.Count)
                {
                    int numInputsRemaining = input.Count - rangeStart;
                    int paddingAmount      = batchSize - numInputsRemaining;
                    batch = input.GetRange(rangeStart, numInputsRemaining);
                    batch.AddRange(input.GetRange(0, paddingAmount));
                }
                else
                {
                    batch = input.GetRange(rangeStart, batchSize);
                }
                var start = HighResolutionClock.UtcNow();
                binding.Bind(inputName, batch);
                session.Evaluate(binding, "");
                var stop     = HighResolutionClock.UtcNow();
                var duration = HighResolutionClock.DurationInMs(start, stop);
                totalDuration += duration;
            }
            return(totalDuration);
        }
Exemple #5
0
#pragma warning disable CA1416 // Validate platform compatibility
        private void ApplyEffects(bool recreateSession = true)
        {
            if (!initialized_ || decoder_ == null)
            {
                return;
            }

            PerformanceMetricsMonitor.ClearLog();

            if (recreateSession)
            {
                RecreateSessions();
            }

            // TensorizeWithVideoFrame();

            long start, stop;

            // Tensorize
            start = HighResolutionClock.UtcNow();
            var pixelDataProvider = decoder_.GetPixelDataAsync().GetAwaiter().GetResult();
            var bytes             = pixelDataProvider.DetachPixelData();
            var buffer            = bytes.AsBuffer(); // Does this do a copy??
            var inputRawTensor    = TensorUInt8Bit.CreateFromBuffer(new long[] { 1, buffer.Length }, buffer);
            // 3 channel NCHW
            var nextOutputShape      = new long[] { 1, 3, currentImageHeight_, currentImageWidth_ };
            var intermediateTensor   = TensorFloat.Create(nextOutputShape);
            var tensorizationBinding = Evaluate(tensorizationSession_, inputRawTensor, intermediateTensor);

            stop = HighResolutionClock.UtcNow();
            var tensorizationDuration = HighResolutionClock.DurationInMs(start, stop);

            // Resize
            start = HighResolutionClock.UtcNow();
            TensorFloat          resizeOutput  = null;
            LearningModelBinding resizeBinding = null;

            if (resizeEffectSession_ != null)
            {
                nextOutputShape    = new long[] { 1, 3, 224, 224 };
                resizeOutput       = TensorFloat.Create(nextOutputShape);
                resizeBinding      = Evaluate(resizeEffectSession_, intermediateTensor, resizeOutput);
                intermediateTensor = resizeOutput;
            }
            stop = HighResolutionClock.UtcNow();
            var resizeDuration = HighResolutionClock.DurationInMs(start, stop);

            // Pixel Swizzle
            start = HighResolutionClock.UtcNow();
            TensorFloat          swizzleOutput  = null;
            LearningModelBinding swizzleBinding = null;

            if (pixelSwizzleEffectSession_ != null)
            {
                swizzleOutput      = TensorFloat.Create(nextOutputShape);
                swizzleBinding     = Evaluate(pixelSwizzleEffectSession_, intermediateTensor, swizzleOutput);
                intermediateTensor = swizzleOutput;
            }
            stop = HighResolutionClock.UtcNow();
            var swizzleDuration = HighResolutionClock.DurationInMs(start, stop);

            // Blur
            start = HighResolutionClock.UtcNow();
            TensorFloat          blurOutput  = null;
            LearningModelBinding blurBinding = null;

            if (blurSharpenEffectSession_ != null)
            {
                blurOutput         = TensorFloat.Create(nextOutputShape);
                blurBinding        = Evaluate(blurSharpenEffectSession_, intermediateTensor, blurOutput);
                intermediateTensor = blurOutput;
            }
            stop = HighResolutionClock.UtcNow();
            var blurDuration = HighResolutionClock.DurationInMs(start, stop);

            // Contrast
            start = HighResolutionClock.UtcNow();
            TensorFloat          contrastOutput  = null;
            LearningModelBinding contrastBinding = null;

            if (contrastEffectSession_ != null)
            {
                contrastOutput     = TensorFloat.Create(nextOutputShape);
                contrastBinding    = EvaluateContrastAndBrightnessSession(intermediateTensor, contrastOutput);
                intermediateTensor = contrastOutput;
            }
            stop = HighResolutionClock.UtcNow();
            var contrastDuration = HighResolutionClock.DurationInMs(start, stop);

            // Artistic Effects
            start = HighResolutionClock.UtcNow();
            LearningModelBinding artistiicEffectsBinding = null;

            if (artisticEffectsEffectSession_ != null)
            {
                var output = TensorFloat.Create(nextOutputShape);
                artistiicEffectsBinding = Evaluate(artisticEffectsEffectSession_, intermediateTensor, output);
                intermediateTensor      = output;
            }
            stop = HighResolutionClock.UtcNow();
            var artisticEffectsDuration = HighResolutionClock.DurationInMs(start, stop);

            // Orientation
            start = HighResolutionClock.UtcNow();
            TensorFloat          orientationOutput  = null;
            LearningModelBinding orientationBinding = null;

            if (orientationEffectSession_ != null)
            {
                var orientationEffect = (OrientationPicker.SelectedItem as OrientationViewModel).Tag;
                if (orientationEffect == Effect.RotateLeft90 ||
                    orientationEffect == Effect.RotateRight90)
                {
                    nextOutputShape   = new long[] { 1, 3, nextOutputShape[3], nextOutputShape[2] };
                    orientationOutput = TensorFloat.Create(nextOutputShape);
                }
                else
                {
                    orientationOutput = TensorFloat.Create(nextOutputShape);
                }
                orientationBinding = Evaluate(orientationEffectSession_, intermediateTensor, orientationOutput);
                intermediateTensor = orientationOutput;
            }
            stop = HighResolutionClock.UtcNow();
            var orientationDuration = HighResolutionClock.DurationInMs(start, stop);

            // Detensorize
            start = HighResolutionClock.UtcNow();
            var shape = intermediateTensor.Shape;
            var n     = (int)shape[0];
            var c     = (int)shape[1];
            var h     = (int)shape[2];
            var w     = (int)shape[3];

            // Rather than writing the data into the software bitmap ourselves from a Tensor (which may be on the gpu)
            // we call an indentity model to move the gpu memory back to the cpu via WinML de-tensorization.
            var outputImage = new SoftwareBitmap(BitmapPixelFormat.Bgra8, w, h, BitmapAlphaMode.Premultiplied);
            var outputFrame = VideoFrame.CreateWithSoftwareBitmap(outputImage);

            var descriptor        = detensorizationSession_.Model.InputFeatures[0] as TensorFeatureDescriptor;
            var detensorizerShape = descriptor.Shape;

            if (c != detensorizerShape[1] || h != detensorizerShape[2] || w != detensorizerShape[3])
            {
                detensorizationSession_ = CreateLearningModelSession(TensorizationModels.IdentityNCHW(n, c, h, w));
            }
            var detensorizationBinding = Evaluate(detensorizationSession_, intermediateTensor, outputFrame, true);

            stop = HighResolutionClock.UtcNow();
            var detensorizationDuration = HighResolutionClock.DurationInMs(start, stop);

            // Render
            var softwareBitmap = outputFrame.SoftwareBitmap;

            RenderingHelpers.BindSoftwareBitmapToImageControl(InputImage, softwareBitmap);

            PerformanceMetricsMonitor.Log("Tensorize", tensorizationDuration);
            PerformanceMetricsMonitor.Log("Resize Effect", resizeDuration);
            PerformanceMetricsMonitor.Log("Swizzle Effect", swizzleDuration);
            PerformanceMetricsMonitor.Log("Blur Effect", blurDuration);
            PerformanceMetricsMonitor.Log("Contrast Effect", contrastDuration);
            PerformanceMetricsMonitor.Log("Artistic Effect", artisticEffectsDuration);
            PerformanceMetricsMonitor.Log("Orientation Effect", orientationDuration);
            PerformanceMetricsMonitor.Log("Detensorize", detensorizationDuration);
        }