private void RecreateSessions()
{
tensorizationSession_?.Dispose();
tensorizationSession_ =
CreateLearningModelSession(
TensorizationModels.ReshapeFlatBufferToNCHW(
1,
4,
currentImageHeight_,
currentImageWidth_));
resizeEffectSession_?.Dispose();
resizeEffectSession_ = GetEffect(ResizeToggleSplitButton, ResizePicker);
pixelSwizzleEffectSession_?.Dispose();
pixelSwizzleEffectSession_ = GetPixelSwizzleEffect();
blurSharpenEffectSession_?.Dispose();
blurSharpenEffectSession_ = GetEffect(BlurSharpenToggleSplitButton, BlurSharpenPicker);
contrastEffectSession_?.Dispose();
contrastEffectSession_ = ContrastToggleSplitButton.IsChecked ?
CreateLearningModelSession(TensorizationModels.AdjustBrightnessAndContrast(
1,
3,
resizeEffectSession_ != null ? 224 : currentImageHeight_,
resizeEffectSession_ != null ? 224 : currentImageWidth_)) :
null;
artisticEffectsEffectSession_?.Dispose();
artisticEffectsEffectSession_ = GetEffect(ArtisticEffectsToggleSplitButton, ArtisticEffectsPicker);
orientationEffectSession_?.Dispose();
orientationEffectSession_ = GetOrientationEffect();
shapeSession_?.Dispose();
shapeSession_ = CreateLearningModelSession(TensorizationModels.ShapeNCHW(1, 3, currentImageHeight_, currentImageWidth_));
detensorizationSession_?.Dispose();
detensorizationSession_ = CreateLearningModelSession(TensorizationModels.IdentityNCHW(
1,
3,
resizeEffectSession_ != null ? 224 : currentImageHeight_,
resizeEffectSession_ != null ? 224 : currentImageWidth_));
}
#pragma warning disable CA1416 // Validate platform compatibility
private void ApplyEffects(bool recreateSession = true)
{
if (!initialized_ || decoder_ == null)
{
return;
}
PerformanceMetricsMonitor.ClearLog();
if (recreateSession)
{
RecreateSessions();
}
// TensorizeWithVideoFrame();
long start, stop;
// Tensorize
start = HighResolutionClock.UtcNow();
var pixelDataProvider = decoder_.GetPixelDataAsync().GetAwaiter().GetResult();
var bytes = pixelDataProvider.DetachPixelData();
var buffer = bytes.AsBuffer(); // Does this do a copy??
var inputRawTensor = TensorUInt8Bit.CreateFromBuffer(new long[] { 1, buffer.Length }, buffer);
// 3 channel NCHW
var nextOutputShape = new long[] { 1, 3, currentImageHeight_, currentImageWidth_ };
var intermediateTensor = TensorFloat.Create(nextOutputShape);
var tensorizationBinding = Evaluate(tensorizationSession_, inputRawTensor, intermediateTensor);
stop = HighResolutionClock.UtcNow();
var tensorizationDuration = HighResolutionClock.DurationInMs(start, stop);
// Resize
start = HighResolutionClock.UtcNow();
TensorFloat resizeOutput = null;
LearningModelBinding resizeBinding = null;
if (resizeEffectSession_ != null)
{
nextOutputShape = new long[] { 1, 3, 224, 224 };
resizeOutput = TensorFloat.Create(nextOutputShape);
resizeBinding = Evaluate(resizeEffectSession_, intermediateTensor, resizeOutput);
intermediateTensor = resizeOutput;
}
stop = HighResolutionClock.UtcNow();
var resizeDuration = HighResolutionClock.DurationInMs(start, stop);
// Pixel Swizzle
start = HighResolutionClock.UtcNow();
TensorFloat swizzleOutput = null;
LearningModelBinding swizzleBinding = null;
if (pixelSwizzleEffectSession_ != null)
{
swizzleOutput = TensorFloat.Create(nextOutputShape);
swizzleBinding = Evaluate(pixelSwizzleEffectSession_, intermediateTensor, swizzleOutput);
intermediateTensor = swizzleOutput;
}
stop = HighResolutionClock.UtcNow();
var swizzleDuration = HighResolutionClock.DurationInMs(start, stop);
// Blur
start = HighResolutionClock.UtcNow();
TensorFloat blurOutput = null;
LearningModelBinding blurBinding = null;
if (blurSharpenEffectSession_ != null)
{
blurOutput = TensorFloat.Create(nextOutputShape);
blurBinding = Evaluate(blurSharpenEffectSession_, intermediateTensor, blurOutput);
intermediateTensor = blurOutput;
}
stop = HighResolutionClock.UtcNow();
var blurDuration = HighResolutionClock.DurationInMs(start, stop);
// Contrast
start = HighResolutionClock.UtcNow();
TensorFloat contrastOutput = null;
LearningModelBinding contrastBinding = null;
if (contrastEffectSession_ != null)
{
contrastOutput = TensorFloat.Create(nextOutputShape);
contrastBinding = EvaluateContrastAndBrightnessSession(intermediateTensor, contrastOutput);
intermediateTensor = contrastOutput;
}
stop = HighResolutionClock.UtcNow();
var contrastDuration = HighResolutionClock.DurationInMs(start, stop);
// Artistic Effects
start = HighResolutionClock.UtcNow();
LearningModelBinding artistiicEffectsBinding = null;
if (artisticEffectsEffectSession_ != null)
{
var output = TensorFloat.Create(nextOutputShape);
artistiicEffectsBinding = Evaluate(artisticEffectsEffectSession_, intermediateTensor, output);
intermediateTensor = output;
}
stop = HighResolutionClock.UtcNow();
var artisticEffectsDuration = HighResolutionClock.DurationInMs(start, stop);
// Orientation
start = HighResolutionClock.UtcNow();
TensorFloat orientationOutput = null;
LearningModelBinding orientationBinding = null;
if (orientationEffectSession_ != null)
{
var orientationEffect = (OrientationPicker.SelectedItem as OrientationViewModel).Tag;
if (orientationEffect == Effect.RotateLeft90 ||
orientationEffect == Effect.RotateRight90)
{
nextOutputShape = new long[] { 1, 3, nextOutputShape[3], nextOutputShape[2] };
orientationOutput = TensorFloat.Create(nextOutputShape);
}
else
{
orientationOutput = TensorFloat.Create(nextOutputShape);
}
orientationBinding = Evaluate(orientationEffectSession_, intermediateTensor, orientationOutput);
intermediateTensor = orientationOutput;
}
stop = HighResolutionClock.UtcNow();
var orientationDuration = HighResolutionClock.DurationInMs(start, stop);
// Detensorize
start = HighResolutionClock.UtcNow();
var shape = intermediateTensor.Shape;
var n = (int)shape[0];
var c = (int)shape[1];
var h = (int)shape[2];
var w = (int)shape[3];
// Rather than writing the data into the software bitmap ourselves from a Tensor (which may be on the gpu)
// we call an indentity model to move the gpu memory back to the cpu via WinML de-tensorization.
var outputImage = new SoftwareBitmap(BitmapPixelFormat.Bgra8, w, h, BitmapAlphaMode.Premultiplied);
var outputFrame = VideoFrame.CreateWithSoftwareBitmap(outputImage);
var descriptor = detensorizationSession_.Model.InputFeatures[0] as TensorFeatureDescriptor;
var detensorizerShape = descriptor.Shape;
if (c != detensorizerShape[1] || h != detensorizerShape[2] || w != detensorizerShape[3])
{
detensorizationSession_ = CreateLearningModelSession(TensorizationModels.IdentityNCHW(n, c, h, w));
}
var detensorizationBinding = Evaluate(detensorizationSession_, intermediateTensor, outputFrame, true);
stop = HighResolutionClock.UtcNow();
var detensorizationDuration = HighResolutionClock.DurationInMs(start, stop);
// Render
var softwareBitmap = outputFrame.SoftwareBitmap;
RenderingHelpers.BindSoftwareBitmapToImageControl(InputImage, softwareBitmap);
PerformanceMetricsMonitor.Log("Tensorize", tensorizationDuration);
PerformanceMetricsMonitor.Log("Resize Effect", resizeDuration);
PerformanceMetricsMonitor.Log("Swizzle Effect", swizzleDuration);
PerformanceMetricsMonitor.Log("Blur Effect", blurDuration);
PerformanceMetricsMonitor.Log("Contrast Effect", contrastDuration);
PerformanceMetricsMonitor.Log("Artistic Effect", artisticEffectsDuration);
PerformanceMetricsMonitor.Log("Orientation Effect", orientationDuration);
PerformanceMetricsMonitor.Log("Detensorize", detensorizationDuration);
}
