private (IEnumerable <string>, IReadOnlyList <float>) Classify(Image <Bgra32> image, float angle)
{
long start, stop;
PerformanceMetricsMonitor.ClearLog();
// Tensorize
start = HighResolutionClock.UtcNow();
image.Mutate(ctx => ctx.Rotate(angle));
var resizeOptions = new ResizeOptions()
{
Mode = ResizeMode.Crop,
Size = new SixLabors.ImageSharp.Size((int)Width, (int)Height)
};
image.Mutate(ctx => ctx.Resize(resizeOptions));
object input = image.AsTensor();
var tensorizationResults = Evaluate(_tensorizationSession, input);
object tensorizedOutput = tensorizationResults.Outputs.First().Value;
stop = HighResolutionClock.UtcNow();
var tensorizeDuration = HighResolutionClock.DurationInMs(start, stop);
// Inference
start = HighResolutionClock.UtcNow();
var inferenceResults = Evaluate(_inferenceSession, tensorizedOutput);
var inferenceOutput = inferenceResults.Outputs.First().Value;
stop = HighResolutionClock.UtcNow();
var inferenceDuration = HighResolutionClock.DurationInMs(start, stop);
// PostProcess
start = HighResolutionClock.UtcNow();
var postProcessedOutputs = Evaluate(_postProcessingSession, inferenceOutput);
var topKValues = (TensorFloat)postProcessedOutputs.Outputs["TopKValues"];
var topKIndices = (TensorInt64Bit)postProcessedOutputs.Outputs["TopKIndices"];
// Return results
var probabilities = topKValues.GetAsVectorView();
var indices = topKIndices.GetAsVectorView();
var labels = indices.Select((index) => ClassificationLabels.ImageNet[index]);
stop = HighResolutionClock.UtcNow();
var postProcessDuration = HighResolutionClock.DurationInMs(start, stop);
PerformanceMetricsMonitor.Log("Tensorize", tensorizeDuration);
PerformanceMetricsMonitor.Log("Pre-process", 0);
PerformanceMetricsMonitor.Log("Inference", inferenceDuration);
PerformanceMetricsMonitor.Log("Post-process", postProcessDuration);
RenderingHelpers.BindSoftwareBitmapToImageControl(InputImage, image.AsSoftwareBitmap());
image.Dispose();
return(labels, probabilities);
}
private (IEnumerable <string>, IReadOnlyList <float>) Classify(WinMLSamplesGalleryNative.OpenCVImage image)
{
long start, stop;
PerformanceMetricsMonitor.ClearLog();
start = HighResolutionClock.UtcNow();
object input = image.AsTensor();
stop = HighResolutionClock.UtcNow();
var pixelAccessDuration = HighResolutionClock.DurationInMs(start, stop);
// Tensorize
start = HighResolutionClock.UtcNow();
object tensorizedOutput = input;
var tensorizationResults = Evaluate(_tensorizationSession, input);
tensorizedOutput = tensorizationResults.Outputs.First().Value;
stop = HighResolutionClock.UtcNow();
var tensorizeDuration = HighResolutionClock.DurationInMs(start, stop);
// Inference
start = HighResolutionClock.UtcNow();
var inferenceResults = Evaluate(_inferenceSession, tensorizedOutput);
var inferenceOutput = inferenceResults.Outputs.First().Value;
stop = HighResolutionClock.UtcNow();
var inferenceDuration = HighResolutionClock.DurationInMs(start, stop);
// PostProcess
start = HighResolutionClock.UtcNow();
var postProcessedOutputs = Evaluate(_postProcessingSession, inferenceOutput);
var topKValues = (TensorFloat)postProcessedOutputs.Outputs["TopKValues"];
var topKIndices = (TensorInt64Bit)postProcessedOutputs.Outputs["TopKIndices"];
// Return results
var probabilities = topKValues.GetAsVectorView();
var indices = topKIndices.GetAsVectorView();
var labels = indices.Select((index) => ClassificationLabels.ImageNet[index]);
stop = HighResolutionClock.UtcNow();
var postProcessDuration = HighResolutionClock.DurationInMs(start, stop);
PerformanceMetricsMonitor.Log("Pixel Access (CPU)", pixelAccessDuration);
PerformanceMetricsMonitor.Log("Tensorize", tensorizeDuration);
PerformanceMetricsMonitor.Log("Pre-process", 0);
PerformanceMetricsMonitor.Log("Inference", inferenceDuration);
PerformanceMetricsMonitor.Log("Post-process", postProcessDuration);
return(labels, probabilities);
}
private static float Evaluate(LearningModelSession session, List <VideoFrame> input)
{
string inputName = session.Model.InputFeatures[0].Name;
float totalDuration = 0;
var binding = new LearningModelBinding(session);
for (int j = 0; j < input.Count; j++)
{
if (navigatingAwayFromPage)
{
break;
}
var start = HighResolutionClock.UtcNow();
binding.Bind(inputName, input[j]);
session.Evaluate(binding, "");
var stop = HighResolutionClock.UtcNow();
var duration = HighResolutionClock.DurationInMs(start, stop);
totalDuration += duration;
}
return(totalDuration);
}
private static float EvaluateBatched(LearningModelSession session, List <VideoFrame> input, int batchSize)
{
int numBatches = (int)Math.Ceiling((Decimal)input.Count / batchSize);
string inputName = session.Model.InputFeatures[0].Name;
float totalDuration = 0;
var binding = new LearningModelBinding(session);
for (int i = 0; i < numBatches; i++)
{
if (navigatingAwayFromPage)
{
break;
}
int rangeStart = batchSize * i;
List <VideoFrame> batch;
// Add padding to the last batch if necessary
if (rangeStart + batchSize > input.Count)
{
int numInputsRemaining = input.Count - rangeStart;
int paddingAmount = batchSize - numInputsRemaining;
batch = input.GetRange(rangeStart, numInputsRemaining);
batch.AddRange(input.GetRange(0, paddingAmount));
}
else
{
batch = input.GetRange(rangeStart, batchSize);
}
var start = HighResolutionClock.UtcNow();
binding.Bind(inputName, batch);
session.Evaluate(binding, "");
var stop = HighResolutionClock.UtcNow();
var duration = HighResolutionClock.DurationInMs(start, stop);
totalDuration += duration;
}
return(totalDuration);
}
#pragma warning disable CA1416 // Validate platform compatibility
private void ApplyEffects(bool recreateSession = true)
{
if (!initialized_ || decoder_ == null)
{
return;
}
PerformanceMetricsMonitor.ClearLog();
if (recreateSession)
{
RecreateSessions();
}
// TensorizeWithVideoFrame();
long start, stop;
// Tensorize
start = HighResolutionClock.UtcNow();
var pixelDataProvider = decoder_.GetPixelDataAsync().GetAwaiter().GetResult();
var bytes = pixelDataProvider.DetachPixelData();
var buffer = bytes.AsBuffer(); // Does this do a copy??
var inputRawTensor = TensorUInt8Bit.CreateFromBuffer(new long[] { 1, buffer.Length }, buffer);
// 3 channel NCHW
var nextOutputShape = new long[] { 1, 3, currentImageHeight_, currentImageWidth_ };
var intermediateTensor = TensorFloat.Create(nextOutputShape);
var tensorizationBinding = Evaluate(tensorizationSession_, inputRawTensor, intermediateTensor);
stop = HighResolutionClock.UtcNow();
var tensorizationDuration = HighResolutionClock.DurationInMs(start, stop);
// Resize
start = HighResolutionClock.UtcNow();
TensorFloat resizeOutput = null;
LearningModelBinding resizeBinding = null;
if (resizeEffectSession_ != null)
{
nextOutputShape = new long[] { 1, 3, 224, 224 };
resizeOutput = TensorFloat.Create(nextOutputShape);
resizeBinding = Evaluate(resizeEffectSession_, intermediateTensor, resizeOutput);
intermediateTensor = resizeOutput;
}
stop = HighResolutionClock.UtcNow();
var resizeDuration = HighResolutionClock.DurationInMs(start, stop);
// Pixel Swizzle
start = HighResolutionClock.UtcNow();
TensorFloat swizzleOutput = null;
LearningModelBinding swizzleBinding = null;
if (pixelSwizzleEffectSession_ != null)
{
swizzleOutput = TensorFloat.Create(nextOutputShape);
swizzleBinding = Evaluate(pixelSwizzleEffectSession_, intermediateTensor, swizzleOutput);
intermediateTensor = swizzleOutput;
}
stop = HighResolutionClock.UtcNow();
var swizzleDuration = HighResolutionClock.DurationInMs(start, stop);
// Blur
start = HighResolutionClock.UtcNow();
TensorFloat blurOutput = null;
LearningModelBinding blurBinding = null;
if (blurSharpenEffectSession_ != null)
{
blurOutput = TensorFloat.Create(nextOutputShape);
blurBinding = Evaluate(blurSharpenEffectSession_, intermediateTensor, blurOutput);
intermediateTensor = blurOutput;
}
stop = HighResolutionClock.UtcNow();
var blurDuration = HighResolutionClock.DurationInMs(start, stop);
// Contrast
start = HighResolutionClock.UtcNow();
TensorFloat contrastOutput = null;
LearningModelBinding contrastBinding = null;
if (contrastEffectSession_ != null)
{
contrastOutput = TensorFloat.Create(nextOutputShape);
contrastBinding = EvaluateContrastAndBrightnessSession(intermediateTensor, contrastOutput);
intermediateTensor = contrastOutput;
}
stop = HighResolutionClock.UtcNow();
var contrastDuration = HighResolutionClock.DurationInMs(start, stop);
// Artistic Effects
start = HighResolutionClock.UtcNow();
LearningModelBinding artistiicEffectsBinding = null;
if (artisticEffectsEffectSession_ != null)
{
var output = TensorFloat.Create(nextOutputShape);
artistiicEffectsBinding = Evaluate(artisticEffectsEffectSession_, intermediateTensor, output);
intermediateTensor = output;
}
stop = HighResolutionClock.UtcNow();
var artisticEffectsDuration = HighResolutionClock.DurationInMs(start, stop);
// Orientation
start = HighResolutionClock.UtcNow();
TensorFloat orientationOutput = null;
LearningModelBinding orientationBinding = null;
if (orientationEffectSession_ != null)
{
var orientationEffect = (OrientationPicker.SelectedItem as OrientationViewModel).Tag;
if (orientationEffect == Effect.RotateLeft90 ||
orientationEffect == Effect.RotateRight90)
{
nextOutputShape = new long[] { 1, 3, nextOutputShape[3], nextOutputShape[2] };
orientationOutput = TensorFloat.Create(nextOutputShape);
}
else
{
orientationOutput = TensorFloat.Create(nextOutputShape);
}
orientationBinding = Evaluate(orientationEffectSession_, intermediateTensor, orientationOutput);
intermediateTensor = orientationOutput;
}
stop = HighResolutionClock.UtcNow();
var orientationDuration = HighResolutionClock.DurationInMs(start, stop);
// Detensorize
start = HighResolutionClock.UtcNow();
var shape = intermediateTensor.Shape;
var n = (int)shape[0];
var c = (int)shape[1];
var h = (int)shape[2];
var w = (int)shape[3];
// Rather than writing the data into the software bitmap ourselves from a Tensor (which may be on the gpu)
// we call an indentity model to move the gpu memory back to the cpu via WinML de-tensorization.
var outputImage = new SoftwareBitmap(BitmapPixelFormat.Bgra8, w, h, BitmapAlphaMode.Premultiplied);
var outputFrame = VideoFrame.CreateWithSoftwareBitmap(outputImage);
var descriptor = detensorizationSession_.Model.InputFeatures[0] as TensorFeatureDescriptor;
var detensorizerShape = descriptor.Shape;
if (c != detensorizerShape[1] || h != detensorizerShape[2] || w != detensorizerShape[3])
{
detensorizationSession_ = CreateLearningModelSession(TensorizationModels.IdentityNCHW(n, c, h, w));
}
var detensorizationBinding = Evaluate(detensorizationSession_, intermediateTensor, outputFrame, true);
stop = HighResolutionClock.UtcNow();
var detensorizationDuration = HighResolutionClock.DurationInMs(start, stop);
// Render
var softwareBitmap = outputFrame.SoftwareBitmap;
RenderingHelpers.BindSoftwareBitmapToImageControl(InputImage, softwareBitmap);
PerformanceMetricsMonitor.Log("Tensorize", tensorizationDuration);
PerformanceMetricsMonitor.Log("Resize Effect", resizeDuration);
PerformanceMetricsMonitor.Log("Swizzle Effect", swizzleDuration);
PerformanceMetricsMonitor.Log("Blur Effect", blurDuration);
PerformanceMetricsMonitor.Log("Contrast Effect", contrastDuration);
PerformanceMetricsMonitor.Log("Artistic Effect", artisticEffectsDuration);
PerformanceMetricsMonitor.Log("Orientation Effect", orientationDuration);
PerformanceMetricsMonitor.Log("Detensorize", detensorizationDuration);
}