private (IEnumerable <string>, IReadOnlyList <float>) Classify(Image <Bgra32> image, float angle)
{
long start, stop;
PerformanceMetricsMonitor.ClearLog();
// Tensorize
start = HighResolutionClock.UtcNow();
image.Mutate(ctx => ctx.Rotate(angle));
var resizeOptions = new ResizeOptions()
{
Mode = ResizeMode.Crop,
Size = new SixLabors.ImageSharp.Size((int)Width, (int)Height)
};
image.Mutate(ctx => ctx.Resize(resizeOptions));
object input = image.AsTensor();
var tensorizationResults = Evaluate(_tensorizationSession, input);
object tensorizedOutput = tensorizationResults.Outputs.First().Value;
stop = HighResolutionClock.UtcNow();
var tensorizeDuration = HighResolutionClock.DurationInMs(start, stop);
// Inference
start = HighResolutionClock.UtcNow();
var inferenceResults = Evaluate(_inferenceSession, tensorizedOutput);
var inferenceOutput = inferenceResults.Outputs.First().Value;
stop = HighResolutionClock.UtcNow();
var inferenceDuration = HighResolutionClock.DurationInMs(start, stop);
// PostProcess
start = HighResolutionClock.UtcNow();
var postProcessedOutputs = Evaluate(_postProcessingSession, inferenceOutput);
var topKValues = (TensorFloat)postProcessedOutputs.Outputs["TopKValues"];
var topKIndices = (TensorInt64Bit)postProcessedOutputs.Outputs["TopKIndices"];
// Return results
var probabilities = topKValues.GetAsVectorView();
var indices = topKIndices.GetAsVectorView();
var labels = indices.Select((index) => ClassificationLabels.ImageNet[index]);
stop = HighResolutionClock.UtcNow();
var postProcessDuration = HighResolutionClock.DurationInMs(start, stop);
PerformanceMetricsMonitor.Log("Tensorize", tensorizeDuration);
PerformanceMetricsMonitor.Log("Pre-process", 0);
PerformanceMetricsMonitor.Log("Inference", inferenceDuration);
PerformanceMetricsMonitor.Log("Post-process", postProcessDuration);
RenderingHelpers.BindSoftwareBitmapToImageControl(InputImage, image.AsSoftwareBitmap());
image.Dispose();
return(labels, probabilities);
}
private void SetCurrentImage(string path)
{
if (CurrentImage != null)
{
CurrentImage.Dispose();
}
RotationSlider.IsEnabled = true;
CurrentImage = SixLabors.ImageSharp.Image.Load <Bgra32>(path);
RenderingHelpers.BindSoftwareBitmapToImageControl(InputImage, CurrentImage.AsSoftwareBitmap());
}
private void TryPerformInference(bool reloadImages = true)
{
if (CurrentImagePath != null)
{
if (reloadImages)
{
Original = WinMLSamplesGalleryNative.OpenCVImage.CreateFromPath(CurrentImagePath);
Noisy = WinMLSamplesGalleryNative.OpenCVImage.AddSaltAndPepperNoise(Original);
Denoised = WinMLSamplesGalleryNative.OpenCVImage.DenoiseMedianBlur(Noisy);
var baseImageBitmap = Original.AsSoftwareBitmap();
RenderingHelpers.BindSoftwareBitmapToImageControl(InputImage, baseImageBitmap);
RenderingHelpers.BindSoftwareBitmapToImageControl(NoisyImage, Noisy.AsSoftwareBitmap());
RenderingHelpers.BindSoftwareBitmapToImageControl(DenoisedImage, Denoised.AsSoftwareBitmap());
var tensorizationModel = TensorizationModels.CastResizeAndTranspose11(Height, Width, 1, 3, baseImageBitmap.PixelHeight, baseImageBitmap.PixelWidth, "nearest");
_tensorizationSession = CreateLearningModelSession(tensorizationModel, SelectedDeviceKind);
}
WinMLSamplesGalleryNative.OpenCVImage classificationImage = null;
switch (InferenceChoice)
{
case ClassifyChoice.Original:
classificationImage = Original;
break;
case ClassifyChoice.Noisy:
classificationImage = Noisy;
break;
case ClassifyChoice.Denoised:
classificationImage = Denoised;
break;
}
// Classify
var(labels, probabilities) = Classify(classificationImage);
// Render the classification and probabilities
RenderInferenceResults(labels, probabilities);
}
}
public HtmlString Render()
{
var renderHelp = new RenderingHelpers(this);
try
{
string innerContent = null;
#region do pre-check to see if system is stable & ready
var notReady = new InstallationController().CheckUpgradeMessage(PortalSettings.Current.UserInfo.IsSuperUser);
if (!string.IsNullOrEmpty(notReady))
{
innerContent = renderHelp.DesignErrorMessage(new Exception(notReady), true, "Error - needs admin to fix this", false, false);
}
#endregion
#region check if the content-group exists (sometimes it's missing if a site is being imported and the data isn't in yet
if (ContentBlock.DataIsMissing)
{
if (Environment.Permissions.UserMayEditContent)
{
innerContent = ""; // stop further processing
}
else // end users should see server error as no js-side processing will happen
{
var ex =
new Exception(
"Data is missing - usually when a site is copied but the content / apps have not been imported yet - check 2sxc.org/help?tag=export-import");
innerContent = renderHelp.DesignErrorMessage(ex, true,
"Error - needs admin to fix", false, true);
}
}
#endregion
#region try to render the block or generate the error message
if (innerContent == null)
{
try
{
if (Template != null) // when a content block is still new, there is no definition yet
{
var engine = GetRenderingEngine(InstancePurposes.WebView);
innerContent = engine.Render();
}
else
{
innerContent = "";
}
}
catch (Exception ex)
{
innerContent = renderHelp.DesignErrorMessage(ex, true, "Error rendering template", false, true);
}
}
#endregion
#region Wrap it all up into a nice wrapper tag
var editInfos = JsonConvert.SerializeObject(renderHelp.GetClientInfosAll());
var startTag = (RenderWithDiv
? $"<div class=\"sc-viewport sc-content-block\" data-cb-instance=\"{ContentBlock.ParentId}\" " +
$" data-cb-id=\"{ContentBlock.ContentBlockId}\""
+
(RenderWithEditMetadata
? " data-edit-context=\'" + editInfos + "'"
: "")
+ ">\n"
: "");
var endTag = (RenderWithDiv ? "\n</div>" : "");
string result = startTag + innerContent + endTag;
#endregion
return(new HtmlString(result));
}
catch (Exception ex)
{
// todo: i18n
return(new HtmlString(renderHelp.DesignErrorMessage(ex, true, null, true, true)));
}
}
#pragma warning disable CA1416 // Validate platform compatibility
private void ApplyEffects(bool recreateSession = true)
{
if (!initialized_ || decoder_ == null)
{
return;
}
PerformanceMetricsMonitor.ClearLog();
if (recreateSession)
{
RecreateSessions();
}
// TensorizeWithVideoFrame();
long start, stop;
// Tensorize
start = HighResolutionClock.UtcNow();
var pixelDataProvider = decoder_.GetPixelDataAsync().GetAwaiter().GetResult();
var bytes = pixelDataProvider.DetachPixelData();
var buffer = bytes.AsBuffer(); // Does this do a copy??
var inputRawTensor = TensorUInt8Bit.CreateFromBuffer(new long[] { 1, buffer.Length }, buffer);
// 3 channel NCHW
var nextOutputShape = new long[] { 1, 3, currentImageHeight_, currentImageWidth_ };
var intermediateTensor = TensorFloat.Create(nextOutputShape);
var tensorizationBinding = Evaluate(tensorizationSession_, inputRawTensor, intermediateTensor);
stop = HighResolutionClock.UtcNow();
var tensorizationDuration = HighResolutionClock.DurationInMs(start, stop);
// Resize
start = HighResolutionClock.UtcNow();
TensorFloat resizeOutput = null;
LearningModelBinding resizeBinding = null;
if (resizeEffectSession_ != null)
{
nextOutputShape = new long[] { 1, 3, 224, 224 };
resizeOutput = TensorFloat.Create(nextOutputShape);
resizeBinding = Evaluate(resizeEffectSession_, intermediateTensor, resizeOutput);
intermediateTensor = resizeOutput;
}
stop = HighResolutionClock.UtcNow();
var resizeDuration = HighResolutionClock.DurationInMs(start, stop);
// Pixel Swizzle
start = HighResolutionClock.UtcNow();
TensorFloat swizzleOutput = null;
LearningModelBinding swizzleBinding = null;
if (pixelSwizzleEffectSession_ != null)
{
swizzleOutput = TensorFloat.Create(nextOutputShape);
swizzleBinding = Evaluate(pixelSwizzleEffectSession_, intermediateTensor, swizzleOutput);
intermediateTensor = swizzleOutput;
}
stop = HighResolutionClock.UtcNow();
var swizzleDuration = HighResolutionClock.DurationInMs(start, stop);
// Blur
start = HighResolutionClock.UtcNow();
TensorFloat blurOutput = null;
LearningModelBinding blurBinding = null;
if (blurSharpenEffectSession_ != null)
{
blurOutput = TensorFloat.Create(nextOutputShape);
blurBinding = Evaluate(blurSharpenEffectSession_, intermediateTensor, blurOutput);
intermediateTensor = blurOutput;
}
stop = HighResolutionClock.UtcNow();
var blurDuration = HighResolutionClock.DurationInMs(start, stop);
// Contrast
start = HighResolutionClock.UtcNow();
TensorFloat contrastOutput = null;
LearningModelBinding contrastBinding = null;
if (contrastEffectSession_ != null)
{
contrastOutput = TensorFloat.Create(nextOutputShape);
contrastBinding = EvaluateContrastAndBrightnessSession(intermediateTensor, contrastOutput);
intermediateTensor = contrastOutput;
}
stop = HighResolutionClock.UtcNow();
var contrastDuration = HighResolutionClock.DurationInMs(start, stop);
// Artistic Effects
start = HighResolutionClock.UtcNow();
LearningModelBinding artistiicEffectsBinding = null;
if (artisticEffectsEffectSession_ != null)
{
var output = TensorFloat.Create(nextOutputShape);
artistiicEffectsBinding = Evaluate(artisticEffectsEffectSession_, intermediateTensor, output);
intermediateTensor = output;
}
stop = HighResolutionClock.UtcNow();
var artisticEffectsDuration = HighResolutionClock.DurationInMs(start, stop);
// Orientation
start = HighResolutionClock.UtcNow();
TensorFloat orientationOutput = null;
LearningModelBinding orientationBinding = null;
if (orientationEffectSession_ != null)
{
var orientationEffect = (OrientationPicker.SelectedItem as OrientationViewModel).Tag;
if (orientationEffect == Effect.RotateLeft90 ||
orientationEffect == Effect.RotateRight90)
{
nextOutputShape = new long[] { 1, 3, nextOutputShape[3], nextOutputShape[2] };
orientationOutput = TensorFloat.Create(nextOutputShape);
}
else
{
orientationOutput = TensorFloat.Create(nextOutputShape);
}
orientationBinding = Evaluate(orientationEffectSession_, intermediateTensor, orientationOutput);
intermediateTensor = orientationOutput;
}
stop = HighResolutionClock.UtcNow();
var orientationDuration = HighResolutionClock.DurationInMs(start, stop);
// Detensorize
start = HighResolutionClock.UtcNow();
var shape = intermediateTensor.Shape;
var n = (int)shape[0];
var c = (int)shape[1];
var h = (int)shape[2];
var w = (int)shape[3];
// Rather than writing the data into the software bitmap ourselves from a Tensor (which may be on the gpu)
// we call an indentity model to move the gpu memory back to the cpu via WinML de-tensorization.
var outputImage = new SoftwareBitmap(BitmapPixelFormat.Bgra8, w, h, BitmapAlphaMode.Premultiplied);
var outputFrame = VideoFrame.CreateWithSoftwareBitmap(outputImage);
var descriptor = detensorizationSession_.Model.InputFeatures[0] as TensorFeatureDescriptor;
var detensorizerShape = descriptor.Shape;
if (c != detensorizerShape[1] || h != detensorizerShape[2] || w != detensorizerShape[3])
{
detensorizationSession_ = CreateLearningModelSession(TensorizationModels.IdentityNCHW(n, c, h, w));
}
var detensorizationBinding = Evaluate(detensorizationSession_, intermediateTensor, outputFrame, true);
stop = HighResolutionClock.UtcNow();
var detensorizationDuration = HighResolutionClock.DurationInMs(start, stop);
// Render
var softwareBitmap = outputFrame.SoftwareBitmap;
RenderingHelpers.BindSoftwareBitmapToImageControl(InputImage, softwareBitmap);
PerformanceMetricsMonitor.Log("Tensorize", tensorizationDuration);
PerformanceMetricsMonitor.Log("Resize Effect", resizeDuration);
PerformanceMetricsMonitor.Log("Swizzle Effect", swizzleDuration);
PerformanceMetricsMonitor.Log("Blur Effect", blurDuration);
PerformanceMetricsMonitor.Log("Contrast Effect", contrastDuration);
PerformanceMetricsMonitor.Log("Artistic Effect", artisticEffectsDuration);
PerformanceMetricsMonitor.Log("Orientation Effect", orientationDuration);
PerformanceMetricsMonitor.Log("Detensorize", detensorizationDuration);
}
public HtmlString Render()
{
var renderHelp = new RenderingHelpers(this);
try
{
string innerContent = null;
#region do pre-check to see if system is stable & ready
var notReady = new InstallationController().CheckUpgradeMessage(PortalSettings.Current.UserInfo.IsSuperUser);
if (!string.IsNullOrEmpty(notReady))
{
innerContent = renderHelp.DesignErrorMessage(new Exception(notReady), true, "Error - needs admin to fix this", false, false);
}
#endregion
#region try to render the block or generate the error message
if (innerContent == null)
{
try
{
if (Template != null) // when a content block is still new, there is no definition yet
{
var engine = GetRenderingEngine(InstancePurposes.WebView);
innerContent = engine.Render();
}
else
{
innerContent = "";
}
}
catch (Exception ex)
{
innerContent = renderHelp.DesignErrorMessage(ex, true, "Error rendering template", false, true);
}
}
#endregion
#region Wrap it all up into a nice wrapper tag
var editInfos = JsonConvert.SerializeObject(renderHelp.GetClientInfosAll());
var startTag = (RenderWithDiv
? $"<div class=\"sc-viewport sc-content-block\" data-cb-instance=\"{ContentBlock.ParentId}\" " +
$" data-cb-id=\"{ContentBlock.ContentBlockId}\""
+
(RenderWithEditMetadata
? " data-edit-context=\'" + editInfos + "'"
: "")
+ ">\n"
: "");
var endTag = (RenderWithDiv ? "\n</div>" : "");
string result = startTag + innerContent + endTag;
#endregion
return(new HtmlString(result));
}
catch (Exception ex)
{
// todo: i18n
return(new HtmlString(renderHelp.DesignErrorMessage(ex, true, null, true, true)));
}
}
