private async Task LoadAndEvaluateModelAsync(VideoFrame _inputFrame, string _modelFileName)
{
LearningModelBinding _binding = null;
VideoFrame _outputFrame = null;
LearningModelSession _session;
try
{
//Load and create the model
if (_model == null)
{
var modelFile =
await StorageFile.GetFileFromApplicationUriAsync(new Uri($"ms-appx:///{_modelFileName}"));
_model = await LearningModel.LoadFromStorageFileAsync(modelFile);
}
// Create the evaluation session with the model
_session = new LearningModelSession(_model);
// Get input and output features of the model
var inputFeatures = _model.InputFeatures.ToList();
var outputFeatures = _model.OutputFeatures.ToList();
// Create binding and then bind input/ output features
_binding = new LearningModelBinding(_session);
_inputImageDescriptor =
inputFeatures.FirstOrDefault(feature => feature.Kind == LearningModelFeatureKind.Tensor) as TensorFeatureDescriptor;
_outputTensorDescriptor =
outputFeatures.FirstOrDefault(feature => feature.Kind == LearningModelFeatureKind.Tensor) as TensorFeatureDescriptor;
TensorFloat outputTensor = TensorFloat.Create(_outputTensorDescriptor.Shape);
ImageFeatureValue imageTensor = ImageFeatureValue.CreateFromVideoFrame(_inputFrame);
// Bind inputs +outputs
_binding.Bind(_inputImageDescriptor.Name, imageTensor);
_binding.Bind(_outputTensorDescriptor.Name, outputTensor);
// Evaluate and get the results
var results = await _session.EvaluateAsync(_binding, "test");
Debug.WriteLine("ResultsEvaluated: " + results.ToString());
var outputTensorList = outputTensor.GetAsVectorView();
var resultsList = new List <float>(outputTensorList.Count);
for (int i = 0; i < outputTensorList.Count; i++)
{
resultsList.Add(outputTensorList[i]);
}
}
catch (Exception ex)
{
Debug.WriteLine($"error: {ex.Message}");
_model = null;
}
}
public async Task <TensorFloat> EvaluateAsync(TensorFloat input)
{
binding.Bind(inName, input);
binding.Bind(outName, TensorFloat.Create(new long[] { 1, 1, InHeight, OutHeight }));
var result = await session.EvaluateAsync(binding, inName);
return(result.Outputs[outName] as TensorFloat);
}
private void SampleInputsGridView_SelectionChanged(object sender, SelectionChangedEventArgs e)
{
var gridView = sender as GridView;
var thumbnail = gridView.SelectedItem as WinMLSamplesGallery.Controls.Thumbnail;
if (thumbnail != null)
{
var image = thumbnail.ImageUri;
var file = StorageFile.GetFileFromApplicationUriAsync(new Uri(image)).GetAwaiter().GetResult();
var softwareBitmap = CreateSoftwareBitmapFromStorageFile(file);
tensorizationSession_ =
CreateLearningModelSession(
TensorizationModels.ReshapeFlatBufferNHWC(
1,
4,
softwareBitmap.PixelHeight,
softwareBitmap.PixelWidth,
416,
416));
// Tensorize
var stream = file.OpenAsync(FileAccessMode.Read).GetAwaiter().GetResult();
var decoder = BitmapDecoder.CreateAsync(stream).GetAwaiter().GetResult();
var bitmap = decoder.GetSoftwareBitmapAsync(BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied).GetAwaiter().GetResult();
var pixelDataProvider = decoder.GetPixelDataAsync().GetAwaiter().GetResult();
var bytes = pixelDataProvider.DetachPixelData();
var buffer = bytes.AsBuffer(); // Does this do a copy??
var inputRawTensor = TensorUInt8Bit.CreateFromBuffer(new long[] { 1, buffer.Length }, buffer);
// 3 channel NCHW
var tensorizeOutput = TensorFloat.Create(new long[] { 1, 416, 416, 3 });
var b = new LearningModelBinding(tensorizationSession_);
b.Bind(tensorizationSession_.Model.InputFeatures[0].Name, inputRawTensor);
b.Bind(tensorizationSession_.Model.OutputFeatures[0].Name, tensorizeOutput);
tensorizationSession_.Evaluate(b, "");
// Resize
var resizeBinding = new LearningModelBinding(_session);
resizeBinding.Bind(_session.Model.InputFeatures[0].Name, tensorizeOutput);
var results = _session.Evaluate(resizeBinding, "");
var output1 = results.Output(0) as TensorFloat;
var data = output1.GetAsVectorView();
var detections = ParseResult(data.ToList <float>().ToArray());
Comparer cp = new Comparer();
detections.Sort(cp);
var final = NMS(detections);
RenderImageInMainPanel(softwareBitmap);
}
}
private static LearningModelEvaluationResult Evaluate(LearningModelSession session, object input)
{
// Create the binding
var binding = new LearningModelBinding(session);
// Create an empty output, that will keep the output resources on the GPU
// It will be chained into a the post processing on the GPU as well
var output = TensorFloat.Create();
// Bind inputs and outputs
// For squeezenet these evaluate to "data", and "squeezenet0_flatten0_reshape0"
string inputName = session.Model.InputFeatures[0].Name;
string outputName = session.Model.OutputFeatures[0].Name;
binding.Bind(inputName, input);
var outputBindProperties = new PropertySet();
outputBindProperties.Add("DisableTensorCpuSync", PropertyValue.CreateBoolean(true));
binding.Bind(outputName, output, outputBindProperties);
// Evaluate
return(session.Evaluate(binding, ""));
}
/// <summary>
/// Evaluate the VideoFrame passed in as arg
/// </summary>
/// <param name="inputFrame"></param>
/// <returns></returns>
private async Task EvaluateVideoFrameAsync(VideoFrame inputFrame)
{
if (inputFrame != null)
{
try
{
StatusBlock.Text = "Binding image...";
// create a binding object from the session
LearningModelBinding binding = new LearningModelBinding(_session);
// bind the input image
ImageFeatureValue imageTensor = ImageFeatureValue.CreateFromVideoFrame(inputFrame);
binding.Bind("data_0", imageTensor);
// temp: there is a bug where winml doesn't allow unbound outputs yet, prebind the output!
{
TensorFeatureDescriptor outputTensorDescription = _model.OutputFeatures.FirstOrDefault(
feature => feature.Name == "softmaxout_1"
) as TensorFeatureDescriptor;
TensorFloat outputTensor = TensorFloat.Create(outputTensorDescription.Shape);
binding.Bind("softmaxout_1", outputTensor);
}
StatusBlock.Text = "Running model...";
int ticks = Environment.TickCount;
// Process the frame with the model
var results = await _session.EvaluateAsync(binding, $"Run { ++_runCount } ");
ticks = Environment.TickCount - ticks;
// retrieve results from evaluation
var resultTensor = results.Outputs["softmaxout_1"] as TensorFloat;
var resultVector = resultTensor.GetAsVectorView();
// Find the top 3 probabilities
List <float> topProbabilities = new List <float>()
{
0.0f, 0.0f, 0.0f
};
List <int> topProbabilityLabelIndexes = new List <int>()
{
0, 0, 0
};
// SqueezeNet returns a list of 1000 options, with probabilities for each, loop through all
for (int i = 0; i < resultVector.Count(); i++)
{
// is it one of the top 3?
for (int j = 0; j < 3; j++)
{
if (resultVector[i] > topProbabilities[j])
{
topProbabilityLabelIndexes[j] = i;
topProbabilities[j] = resultVector[i];
break;
}
}
}
// Display the result
string message = $"Run took { ticks } ticks";
for (int i = 0; i < 3; i++)
{
message += $"\n\"{ _labels[topProbabilityLabelIndexes[i]]}\" with confidence of { topProbabilities[i]}";
}
StatusBlock.Text = message;
}
catch (Exception ex)
{
StatusBlock.Text = $"error: {ex.Message}";
}
ButtonRun.IsEnabled = true;
}
}
private async void Current_SoftwareBitmapFrameCaptured(object sender, SoftwareBitmapEventArgs e)
{
Debug.WriteLine("FrameCaptured");
Debug.WriteLine($"Frame evaluation started {DateTime.Now}");
if (e.SoftwareBitmap != null)
{
BitmapPixelFormat bpf = e.SoftwareBitmap.BitmapPixelFormat;
var uncroppedBitmap = SoftwareBitmap.Convert(e.SoftwareBitmap, BitmapPixelFormat.Nv12);
var faces = await _faceDetector.DetectFacesAsync(uncroppedBitmap);
if (faces.Count > 0)
{
//crop image to focus on face portion
var faceBox = faces[0].FaceBox;
VideoFrame inputFrame = VideoFrame.CreateWithSoftwareBitmap(e.SoftwareBitmap);
VideoFrame tmp = null;
tmp = new VideoFrame(e.SoftwareBitmap.BitmapPixelFormat, (int)(faceBox.Width + faceBox.Width % 2) - 2, (int)(faceBox.Height + faceBox.Height % 2) - 2);
await inputFrame.CopyToAsync(tmp, faceBox, null);
//crop image to fit model input requirements
VideoFrame croppedInputImage = new VideoFrame(BitmapPixelFormat.Gray8, (int)_inputImageDescriptor.Shape[3], (int)_inputImageDescriptor.Shape[2]);
var srcBounds = GetCropBounds(
tmp.SoftwareBitmap.PixelWidth,
tmp.SoftwareBitmap.PixelHeight,
croppedInputImage.SoftwareBitmap.PixelWidth,
croppedInputImage.SoftwareBitmap.PixelHeight);
await tmp.CopyToAsync(croppedInputImage, srcBounds, null);
ImageFeatureValue imageTensor = ImageFeatureValue.CreateFromVideoFrame(croppedInputImage);
_binding = new LearningModelBinding(_session);
TensorFloat outputTensor = TensorFloat.Create(_outputTensorDescriptor.Shape);
List <float> _outputVariableList = new List <float>();
// Bind inputs + outputs
_binding.Bind(_inputImageDescriptor.Name, imageTensor);
_binding.Bind(_outputTensorDescriptor.Name, outputTensor);
// Evaluate results
var results = await _session.EvaluateAsync(_binding, new Guid().ToString());
Debug.WriteLine("ResultsEvaluated: " + results.ToString());
var outputTensorList = outputTensor.GetAsVectorView();
var resultsList = new List <float>(outputTensorList.Count);
for (int i = 0; i < outputTensorList.Count; i++)
{
resultsList.Add(outputTensorList[i]);
}
var softMaxexOutputs = SoftMax(resultsList);
double maxProb = 0;
int maxIndex = 0;
// Comb through the evaluation results
for (int i = 0; i < Constants.POTENTIAL_EMOJI_NAME_LIST.Count(); i++)
{
// Record the dominant emotion probability & its location
if (softMaxexOutputs[i] > maxProb)
{
maxIndex = i;
maxProb = softMaxexOutputs[i];
}
//for evaluations run on the EmotionPage, record info about single specific emotion of interest
if (CurrentEmojis._currentEmoji != null && Constants.POTENTIAL_EMOJI_NAME_LIST[i].Equals(CurrentEmojis._currentEmoji.Name))
{
SoftwareBitmap potentialBestPic;
try
{
potentialBestPic = SoftwareBitmap.Convert(uncroppedBitmap, BitmapPixelFormat.Bgra8);
}
catch (Exception ex)
{
Debug.WriteLine($"Error converting SoftwareBitmap. Details:{ex.Message}. Attempting to continue...");
return;
}
await Windows.ApplicationModel.Core.CoreApplication.MainView.CoreWindow.Dispatcher.RunAsync(CoreDispatcherPriority.Normal,
async() =>
{
// Give user immediate visual feedback by updating success gauge
ScoreUpdated?.Invoke(this, new EmotionPageGaugeScoreEventArgs()
{
Score = softMaxexOutputs[i]
});
// Save original pic for each emotion no matter how bad it is (and record its associated info)
double bestScore = CurrentEmojis._emojis.Emojis[CurrentEmojis._currentEmojiIndex].BestScore;
if (softMaxexOutputs[i] > bestScore)
{
CurrentEmojis._emojis.Emojis[CurrentEmojis._currentEmojiIndex].BestScore = softMaxexOutputs[i];
var source = new SoftwareBitmapSource();
await source.SetBitmapAsync(potentialBestPic);
// Create format of potentialBestPic to be displayed in a gif later
SoftwareBitmap tmpBitmap = potentialBestPic;
WriteableBitmap wb = new WriteableBitmap(tmpBitmap.PixelWidth, tmpBitmap.PixelHeight);
tmpBitmap.CopyToBuffer(wb.PixelBuffer);
CurrentEmojis._emojis.Emojis[CurrentEmojis._currentEmojiIndex].BestPic = source;
CurrentEmojis._emojis.Emojis[CurrentEmojis._currentEmojiIndex].ShowOopsIcon = false;
CurrentEmojis._emojis.Emojis[CurrentEmojis._currentEmojiIndex].BestPicWB = wb;
}
}
);
}
}
Debug.WriteLine($"Probability = {maxProb}, Threshold set to = {Constants.CLASSIFICATION_CERTAINTY_THRESHOLD}, Emotion = {Constants.POTENTIAL_EMOJI_NAME_LIST[maxIndex]}");
// For evaluations run on the MainPage, update the emoji carousel
if (maxProb >= Constants.CLASSIFICATION_CERTAINTY_THRESHOLD)
{
Debug.WriteLine("first page emoji should start to update");
IntelligenceServiceEmotionClassified?.Invoke(this, new ClassifiedEmojiEventArgs(CurrentEmojis._emojis.Emojis[maxIndex]));
}
// Dispose of resources
if (e.SoftwareBitmap != null)
{
e.SoftwareBitmap.Dispose();
e.SoftwareBitmap = null;
}
}
}
IntelligenceServiceProcessingCompleted?.Invoke(this, null);
Debug.WriteLine($"Frame evaluation finished {DateTime.Now}");
}
private async void Current_SoftwareBitmapFrameCaptured(object sender, SoftwareBitmapEventArgs e)
{
Debug.WriteLine("FrameCaptured");
Debug.WriteLine($"Frame evaluation started {DateTime.Now}");
if (e.SoftwareBitmap != null)
{
BitmapPixelFormat bpf = e.SoftwareBitmap.BitmapPixelFormat;
var uncroppedBitmap = SoftwareBitmap.Convert(e.SoftwareBitmap, BitmapPixelFormat.Nv12);
var faces = await _faceDetector.DetectFacesAsync(uncroppedBitmap);
if (faces.Count > 0)
{
//crop image to focus on face portion
var faceBox = faces[0].FaceBox;
VideoFrame inputFrame = VideoFrame.CreateWithSoftwareBitmap(e.SoftwareBitmap);
VideoFrame tmp = null;
tmp = new VideoFrame(e.SoftwareBitmap.BitmapPixelFormat, (int)(faceBox.Width + faceBox.Width % 2) - 2, (int)(faceBox.Height + faceBox.Height % 2) - 2);
await inputFrame.CopyToAsync(tmp, faceBox, null);
//crop image to fit model input requirements
VideoFrame croppedInputImage = new VideoFrame(BitmapPixelFormat.Gray8, (int)_inputImageDescriptor.Shape[3], (int)_inputImageDescriptor.Shape[2]);
var srcBounds = GetCropBounds(
tmp.SoftwareBitmap.PixelWidth,
tmp.SoftwareBitmap.PixelHeight,
croppedInputImage.SoftwareBitmap.PixelWidth,
croppedInputImage.SoftwareBitmap.PixelHeight);
await tmp.CopyToAsync(croppedInputImage, srcBounds, null);
ImageFeatureValue imageTensor = ImageFeatureValue.CreateFromVideoFrame(croppedInputImage);
_binding = new LearningModelBinding(_session);
TensorFloat outputTensor = TensorFloat.Create(_outputTensorDescriptor.Shape);
List <float> _outputVariableList = new List <float>();
// Bind inputs + outputs
_binding.Bind(_inputImageDescriptor.Name, imageTensor);
_binding.Bind(_outputTensorDescriptor.Name, outputTensor);
// Evaluate results
var results = await _session.EvaluateAsync(_binding, new Guid().ToString());
Debug.WriteLine("ResultsEvaluated: " + results.ToString());
var outputTensorList = outputTensor.GetAsVectorView();
var resultsList = new List <float>(outputTensorList.Count);
for (int i = 0; i < outputTensorList.Count; i++)
{
resultsList.Add(outputTensorList[i]);
}
var softMaxexOutputs = SoftMax(resultsList);
double maxProb = 0;
int maxIndex = 0;
// Comb through the evaluation results
for (int i = 0; i < Constants.POTENTIAL_EMOJI_NAME_LIST.Count(); i++)
{
// Record the dominant emotion probability & its location
if (softMaxexOutputs[i] > maxProb)
{
maxIndex = i;
maxProb = softMaxexOutputs[i];
}
}
Debug.WriteLine($"Probability = {maxProb}, Threshold set to = {Constants.CLASSIFICATION_CERTAINTY_THRESHOLD}, Emotion = {Constants.POTENTIAL_EMOJI_NAME_LIST[maxIndex]}");
// For evaluations run on the MainPage, update the emoji carousel
if (maxProb >= Constants.CLASSIFICATION_CERTAINTY_THRESHOLD)
{
Debug.WriteLine("first page emoji should start to update");
IntelligenceServiceEmotionClassified?.Invoke(this, new ClassifiedEmojiEventArgs(CurrentEmojis._emojis.Emojis[maxIndex]));
}
// Dispose of resources
if (e.SoftwareBitmap != null)
{
e.SoftwareBitmap.Dispose();
e.SoftwareBitmap = null;
}
}
}
IntelligenceServiceProcessingCompleted?.Invoke(this, null);
Debug.WriteLine($"Frame evaluation finished {DateTime.Now}");
}
unsafe private TensorFloat CustomTensorize(List <VideoFrame> frameList, List <float> mean, List <float> std, bool toRGB = false)
{
int temp_len = frameList.Count();
SoftwareBitmap softwareBitmap = frameList[0].SoftwareBitmap;
Int32 height = softwareBitmap.PixelHeight;
Int32 width = softwareBitmap.PixelWidth;
BitmapPixelFormat pixelFormat = softwareBitmap.BitmapPixelFormat;
Int32 channels = BitmapPixelFormat.Gray8 == pixelFormat ? 1 : 3;
List <Int64> shape = new List <Int64>()
{
1, temp_len, channels, height, width
}; // B,T,C,H,W
// The channels of image stored in buffer is in order of BGRA-BGRA-BGRA-BGRA.
// Then we transform it to the order of BBBBB....GGGGG....RRRR....AAAA(dropped)
TensorFloat tf = TensorFloat.Create(shape);
byte * pCPUTensorbyte;
float * pCPUTensor;
uint uCapacity;
// The channels of image stored in buffer is in order of BGRA-BGRA-BGRA-BGRA.
// Then we transform it to the order of BBBBB....GGGGG....RRRR....AAAA(dropped)
var tfr = tf.CreateReference();
var tfr2 = (IMemoryBufferByteAccess)tfr;
tfr2.GetBuffer(out pCPUTensorbyte, out uCapacity);
pCPUTensor = (float *)pCPUTensorbyte;
for (Int32 t = 0; t < temp_len; t += 1)
{
VideoFrame frame = frameList[t];
SoftwareBitmap softwareBitmap2 = frame.SoftwareBitmap;
// 1. Get the access to buffer of softwarebitmap
BitmapBuffer spBitmapBuffer = softwareBitmap2.LockBuffer(BitmapBufferAccessMode.Read);
IMemoryBufferReference reference = spBitmapBuffer.CreateReference();
byte *pData;
uint size;
((IMemoryBufferByteAccess)reference).GetBuffer(out pData, out size);
// 2. Transform the data in buffer to a vector of float
var offset = (height * width * channels) * t;
if (BitmapPixelFormat.Bgra8 == pixelFormat)
{
for (UInt32 i = 0; i < size; i += 4)
{
if (toRGB)
{
// suppose the model expects BGR image.
// index 0 is B, 1 is G, 2 is R, 3 is alpha(dropped).
UInt32 pixelInd = i / 4;
pCPUTensor[offset + (height * width * 0) + pixelInd] = (((float)pData[i + 2]) - mean[0]) / std[0];
pCPUTensor[offset + (height * width * 1) + pixelInd] = (((float)pData[i + 1]) - mean[1]) / std[1];
pCPUTensor[offset + (height * width * 2) + pixelInd] = (((float)pData[i + 0]) - mean[2]) / std[2];
}
else
{
// suppose the model expects BGR image.
// index 0 is B, 1 is G, 2 is R, 3 is alpha(dropped).
UInt32 pixelInd = i / 4;
pCPUTensor[offset + (height * width * 0) + pixelInd] = (((float)pData[i + 0]) - mean[0]) / std[0];
pCPUTensor[offset + (height * width * 1) + pixelInd] = (((float)pData[i + 1]) - mean[1]) / std[1];
pCPUTensor[offset + (height * width * 2) + pixelInd] = (((float)pData[i + 2]) - mean[2]) / std[2];
}
}
}
else if (BitmapPixelFormat.Rgba8 == pixelFormat)
{
for (UInt32 i = 0; i < size; i += 4)
{
// suppose the model expects BGR image.
// index 0 is B, 1 is G, 2 is R, 3 is alpha(dropped).
if (toRGB)
{
// suppose the model expects BGR image.
// index 0 is B, 1 is G, 2 is R, 3 is alpha(dropped).
UInt32 pixelInd = i / 4;
pCPUTensor[offset + (height * width * 0) + pixelInd] = (((float)pData[i + 0]) - mean[0]) / std[0];
pCPUTensor[offset + (height * width * 1) + pixelInd] = (((float)pData[i + 1]) - mean[1]) / std[1];
pCPUTensor[offset + (height * width * 2) + pixelInd] = (((float)pData[i + 2]) - mean[2]) / std[2];
}
else
{
UInt32 pixelInd = i / 4;
pCPUTensor[offset + (height * width * 0) + pixelInd] = (((float)pData[i + 2]) - mean[0]) / std[0];
pCPUTensor[offset + (height * width * 1) + pixelInd] = (((float)pData[i + 1]) - mean[1]) / std[1];
pCPUTensor[offset + (height * width * 2) + pixelInd] = (((float)pData[i + 0]) - mean[2]) / std[2];
}
}
}
else if (BitmapPixelFormat.Gray8 == pixelFormat)
{
for (UInt32 i = 0; i < size; i += 4)
{
// suppose the model expects BGR image.
// index 0 is B, 1 is G, 2 is R, 3 is alpha(dropped).
UInt32 pixelInd = i / 4;
float red = (float)pData[i + 2];
float green = (float)pData[i + 1];
float blue = (float)pData[i];
float gray = 0.2126f * red + 0.7152f * green + 0.0722f * blue;
pCPUTensor[offset + pixelInd] = gray;
}
}
}
// to prepend following error, copy to another instance and use it as model input.
// The tensor has outstanding memory buffer references that must be closed prior to evaluation!
TensorFloat ret = TensorFloat.CreateFromIterable(
tf.Shape,
tf.GetAsVectorView());
return(ret);
}
#pragma warning disable CA1416 // Validate platform compatibility
private void ApplyEffects(bool recreateSession = true)
{
if (!initialized_ || decoder_ == null)
{
return;
}
PerformanceMetricsMonitor.ClearLog();
if (recreateSession)
{
RecreateSessions();
}
// TensorizeWithVideoFrame();
long start, stop;
// Tensorize
start = HighResolutionClock.UtcNow();
var pixelDataProvider = decoder_.GetPixelDataAsync().GetAwaiter().GetResult();
var bytes = pixelDataProvider.DetachPixelData();
var buffer = bytes.AsBuffer(); // Does this do a copy??
var inputRawTensor = TensorUInt8Bit.CreateFromBuffer(new long[] { 1, buffer.Length }, buffer);
// 3 channel NCHW
var nextOutputShape = new long[] { 1, 3, currentImageHeight_, currentImageWidth_ };
var intermediateTensor = TensorFloat.Create(nextOutputShape);
var tensorizationBinding = Evaluate(tensorizationSession_, inputRawTensor, intermediateTensor);
stop = HighResolutionClock.UtcNow();
var tensorizationDuration = HighResolutionClock.DurationInMs(start, stop);
// Resize
start = HighResolutionClock.UtcNow();
TensorFloat resizeOutput = null;
LearningModelBinding resizeBinding = null;
if (resizeEffectSession_ != null)
{
nextOutputShape = new long[] { 1, 3, 224, 224 };
resizeOutput = TensorFloat.Create(nextOutputShape);
resizeBinding = Evaluate(resizeEffectSession_, intermediateTensor, resizeOutput);
intermediateTensor = resizeOutput;
}
stop = HighResolutionClock.UtcNow();
var resizeDuration = HighResolutionClock.DurationInMs(start, stop);
// Pixel Swizzle
start = HighResolutionClock.UtcNow();
TensorFloat swizzleOutput = null;
LearningModelBinding swizzleBinding = null;
if (pixelSwizzleEffectSession_ != null)
{
swizzleOutput = TensorFloat.Create(nextOutputShape);
swizzleBinding = Evaluate(pixelSwizzleEffectSession_, intermediateTensor, swizzleOutput);
intermediateTensor = swizzleOutput;
}
stop = HighResolutionClock.UtcNow();
var swizzleDuration = HighResolutionClock.DurationInMs(start, stop);
// Blur
start = HighResolutionClock.UtcNow();
TensorFloat blurOutput = null;
LearningModelBinding blurBinding = null;
if (blurSharpenEffectSession_ != null)
{
blurOutput = TensorFloat.Create(nextOutputShape);
blurBinding = Evaluate(blurSharpenEffectSession_, intermediateTensor, blurOutput);
intermediateTensor = blurOutput;
}
stop = HighResolutionClock.UtcNow();
var blurDuration = HighResolutionClock.DurationInMs(start, stop);
// Contrast
start = HighResolutionClock.UtcNow();
TensorFloat contrastOutput = null;
LearningModelBinding contrastBinding = null;
if (contrastEffectSession_ != null)
{
contrastOutput = TensorFloat.Create(nextOutputShape);
contrastBinding = EvaluateContrastAndBrightnessSession(intermediateTensor, contrastOutput);
intermediateTensor = contrastOutput;
}
stop = HighResolutionClock.UtcNow();
var contrastDuration = HighResolutionClock.DurationInMs(start, stop);
// Artistic Effects
start = HighResolutionClock.UtcNow();
LearningModelBinding artistiicEffectsBinding = null;
if (artisticEffectsEffectSession_ != null)
{
var output = TensorFloat.Create(nextOutputShape);
artistiicEffectsBinding = Evaluate(artisticEffectsEffectSession_, intermediateTensor, output);
intermediateTensor = output;
}
stop = HighResolutionClock.UtcNow();
var artisticEffectsDuration = HighResolutionClock.DurationInMs(start, stop);
// Orientation
start = HighResolutionClock.UtcNow();
TensorFloat orientationOutput = null;
LearningModelBinding orientationBinding = null;
if (orientationEffectSession_ != null)
{
var orientationEffect = (OrientationPicker.SelectedItem as OrientationViewModel).Tag;
if (orientationEffect == Effect.RotateLeft90 ||
orientationEffect == Effect.RotateRight90)
{
nextOutputShape = new long[] { 1, 3, nextOutputShape[3], nextOutputShape[2] };
orientationOutput = TensorFloat.Create(nextOutputShape);
}
else
{
orientationOutput = TensorFloat.Create(nextOutputShape);
}
orientationBinding = Evaluate(orientationEffectSession_, intermediateTensor, orientationOutput);
intermediateTensor = orientationOutput;
}
stop = HighResolutionClock.UtcNow();
var orientationDuration = HighResolutionClock.DurationInMs(start, stop);
// Detensorize
start = HighResolutionClock.UtcNow();
var shape = intermediateTensor.Shape;
var n = (int)shape[0];
var c = (int)shape[1];
var h = (int)shape[2];
var w = (int)shape[3];
// Rather than writing the data into the software bitmap ourselves from a Tensor (which may be on the gpu)
// we call an indentity model to move the gpu memory back to the cpu via WinML de-tensorization.
var outputImage = new SoftwareBitmap(BitmapPixelFormat.Bgra8, w, h, BitmapAlphaMode.Premultiplied);
var outputFrame = VideoFrame.CreateWithSoftwareBitmap(outputImage);
var descriptor = detensorizationSession_.Model.InputFeatures[0] as TensorFeatureDescriptor;
var detensorizerShape = descriptor.Shape;
if (c != detensorizerShape[1] || h != detensorizerShape[2] || w != detensorizerShape[3])
{
detensorizationSession_ = CreateLearningModelSession(TensorizationModels.IdentityNCHW(n, c, h, w));
}
var detensorizationBinding = Evaluate(detensorizationSession_, intermediateTensor, outputFrame, true);
stop = HighResolutionClock.UtcNow();
var detensorizationDuration = HighResolutionClock.DurationInMs(start, stop);
// Render
var softwareBitmap = outputFrame.SoftwareBitmap;
RenderingHelpers.BindSoftwareBitmapToImageControl(InputImage, softwareBitmap);
PerformanceMetricsMonitor.Log("Tensorize", tensorizationDuration);
PerformanceMetricsMonitor.Log("Resize Effect", resizeDuration);
PerformanceMetricsMonitor.Log("Swizzle Effect", swizzleDuration);
PerformanceMetricsMonitor.Log("Blur Effect", blurDuration);
PerformanceMetricsMonitor.Log("Contrast Effect", contrastDuration);
PerformanceMetricsMonitor.Log("Artistic Effect", artisticEffectsDuration);
PerformanceMetricsMonitor.Log("Orientation Effect", orientationDuration);
PerformanceMetricsMonitor.Log("Detensorize", detensorizationDuration);
}
