public async Task <Inceptionv3_convertedOutput> Evaluate(StorageFile file)
{
Inceptionv3_convertedInput tensorInput = new Inceptionv3_convertedInput();
byte[] image = await ResizedImage(file, INPUT_WIDTH, INPUT_HEIGHT);
List <float> input = new List <float>();
List <float> R = new List <float>();
List <float> G = new List <float>();
List <float> B = new List <float>();
for (int j = 0; j < INPUT_HEIGHT; j++)
{
for (int i = 0; i < INPUT_WIDTH; i++)
{
R.Add(GetPixel(image, i, j, INPUT_WIDTH, INPUT_HEIGHT).R / 255f);
G.Add(GetPixel(image, i, j, INPUT_WIDTH, INPUT_HEIGHT).G / 255f);
B.Add(GetPixel(image, i, j, INPUT_WIDTH, INPUT_HEIGHT).B / 255f);
}
}
input.AddRange(R);
input.AddRange(G);
input.AddRange(B);
tensorInput.input_1_0 = TensorFloat.CreateFromArray(new long[] { 1, 256, 256, 3 }, input.ToArray());
return(await Model.EvaluateAsync(tensorInput));
}
private async Task ToImage(TensorFloat tensorFloat, Image image)
{
var pixels = tensorFloat
.GetAsVectorView()
.SelectMany(
f =>
{
byte v = Convert.ToByte(f * 255);
return(new byte[] { v, v, v, 255 });
})
.ToArray();
var writeableBitmap = new WriteableBitmap(320, 320);
// Open a stream to copy the image contents to the WriteableBitmap's pixel buffer
using (Stream stream = writeableBitmap.PixelBuffer.AsStream())
{
await stream.WriteAsync(pixels, 0, pixels.Length);
}
var dest = SoftwareBitmap.CreateCopyFromBuffer(writeableBitmap.PixelBuffer, BitmapPixelFormat.Bgra8, 320, 320, BitmapAlphaMode.Premultiplied);
var destSouce = new SoftwareBitmapSource();
await destSouce.SetBitmapAsync(dest);
image.Source = destSouce;
}
private IList <PredictionModel> Postprocess(TensorFloat predictionOutputs)
{
var(boxes, probs) = ExtractBoxes(predictionOutputs, Anchors);
var result = SuppressNonMaximum(boxes, probs);
// filter to have the max % per detected label
var topLabels = new List <PredictionModel>();
var topValues = new Dictionary <string, float>();
result = result.OrderByDescending(a => a.TagName).ThenByDescending(a => a.Probability).ToList();
foreach (var prediction in result)
{
float value = 0;
if (topValues.ContainsKey(prediction.TagName))
{
value = topValues[prediction.TagName];
}
else
{
topValues.Add(prediction.TagName, prediction.Probability);
}
if (!(value < prediction.Probability))
{
continue;
}
topLabels.Add(prediction);
topValues[prediction.TagName] = prediction.Probability;
}
return(topLabels);
}
private unsafe LearningModelBinding EvaluateContrastAndBrightnessSession(object input, object output)
{
var slope = Math.Tan(ContrastMaxSlider.Value * 3.14159 / 2);
var yintercept = -255 * (ContrastMinSlider.Value * 2 - 1);
if (yintercept < 0)
{
// it was the x-intercept
yintercept = slope * yintercept;
}
var binding = new LearningModelBinding(contrastEffectSession_);
binding.Bind("Input", input);
binding.Bind("Slope", TensorFloat.CreateFromArray(new long[] { 1 }, new float[] { (float)slope }));
binding.Bind("YIntercept", TensorFloat.CreateFromArray(new long[] { 1 }, new float[] { (float)yintercept }));
var outputBindProperties = new PropertySet();
outputBindProperties.Add("DisableTensorCpuSync", PropertyValue.CreateBoolean(true));
binding.Bind("Output", output, outputBindProperties);
EvaluateInternal(contrastEffectSession_, binding);
return(binding);
}
/// <summary>
/// PreProcessing.
/// Converts image data in SoftwareBitmap to TensorFloat.
/// </summary>
public static TensorFloat SoftwareBitmapToTensorFloat(SoftwareBitmap image)
{
int width = image.PixelWidth;
int height = image.PixelHeight;
using (BitmapBuffer buffer = image.LockBuffer(BitmapBufferAccessMode.Read))
{
using (var reference = buffer.CreateReference())
{
// Implementation Reference:
// https://github.com/Microsoft/Windows-Machine-Learning/issues/22
unsafe
{
((IMemoryBufferByteAccess)reference).GetBuffer(out byte *dataInBytes, out uint capacity);
long[] shape = { 1, 3, height, width };
float[] pCPUTensor = new float[3 * width * height];
for (int i = 0; i < capacity; i += 4)
{
int pixelInd = i / 4;
pCPUTensor[pixelInd] = (float)dataInBytes[i];
pCPUTensor[(height * width) + pixelInd] = (float)dataInBytes[i + 1];
pCPUTensor[(height * width * 2) + pixelInd] = (float)dataInBytes[i + 2];
}
float[] processedTensor = NormalizeFloatArray(pCPUTensor);
TensorFloat tensorFloats = TensorFloat.CreateFromArray(shape, processedTensor);
return(tensorFloats);
}
}
}
}
/// <summary>
/// PostProcessing.
/// Generates a list of BBox containing the detected face info.
/// </summary>
/// <param name="boundingBoxCollection">empty list of FaceDetectionRec to store results.</param>
/// <param name="scores">score output of Onnx model.</param>
/// <param name="boxes">box output of Onnx model.</param>
/// <param name="scoreThreshold">threshold of score between 0 and 1 for filtering boxes.</param>
private static void GenerateBBox(
ICollection <FaceDetectionRectangle> boundingBoxCollection,
TensorFloat scores, TensorFloat boxes,
float scoreThreshold)
{
IReadOnlyList <float> vectorBoxes = boxes.GetAsVectorView();
IList <float> boxList = vectorBoxes.ToList();
IReadOnlyList <float> vectorScores = scores.GetAsVectorView();
IList <float> scoreList = vectorScores.ToList();
long numAnchors = scores.Shape[1];
if (numAnchors <= 0)
{
return;
}
for (var i = 0; i < numAnchors; i++)
{
if (scoreList[i * 2 + 1] > scoreThreshold)
{
var rect = new FaceDetectionRectangle
{
X1 = boxList[i * 4] * inputImageDataWidth,
Y1 = boxList[i * 4 + 1] * inputImageDataHeight,
X2 = boxList[i * 4 + 2] * inputImageDataWidth,
Y2 = boxList[i * 4 + 3] * inputImageDataHeight,
Score = Clip(scoreList[i * 2 + 1], 0, 1)
};
boundingBoxCollection.Add(rect);
}
}
}
private IEnumerable <byte> ApplyTensorAsMask(byte[] data, TensorFloat tensorFloat, float cutoff)
{
var tensorData = tensorFloat.GetAsVectorView().ToArray();
for (int i = 0; i < data.Length; i += 4)
{
var alpha = Math.Clamp(tensorData[i / 4], 0, 1);
if (alpha > cutoff)
{
yield return(Convert.ToByte(data[i + 2] * alpha));
yield return(Convert.ToByte(data[i + 1] * alpha));
yield return(Convert.ToByte(data[i + 0] * alpha));
yield return(Convert.ToByte(alpha * 255));
}
else
{
yield return(0);
yield return(0);
yield return(0);
yield return(0);
}
}
}
private async void RecogNumberFromInk()
{
// 从文件加载模型
var modelFile = await StorageFile.GetFileFromApplicationUriAsync(new Uri($"ms-appx:///Assets/mnist.onnx"));
var model = await mnistModel.CreateFromStreamAsync(modelFile);
// 组织输入
var inputArray = await GetInputDataFromInk();
var inputTensor = TensorFloat.CreateFromArray(new List <long> {
784
}, inputArray);
var modelInput = new mnistInput {
port = inputTensor
};
// 推理
var result = await model.EvaluateAsync(modelInput);
// 得到每个数字的得分
var scoreList = result.dense3port.GetAsVectorView().ToList();
// 从输出中取出得分最高的
var max = scoreList.IndexOf(scoreList.Max());
// 显示在控件中
lbResult.Text = max.ToString();
}
private async Task LoadAndEvaluateModelAsync(VideoFrame _inputFrame, string _modelFileName)
{
LearningModelBinding _binding = null;
VideoFrame _outputFrame = null;
LearningModelSession _session;
try
{
//Load and create the model
if (_model == null)
{
var modelFile =
await StorageFile.GetFileFromApplicationUriAsync(new Uri($"ms-appx:///{_modelFileName}"));
_model = await LearningModel.LoadFromStorageFileAsync(modelFile);
}
// Create the evaluation session with the model
_session = new LearningModelSession(_model);
// Get input and output features of the model
var inputFeatures = _model.InputFeatures.ToList();
var outputFeatures = _model.OutputFeatures.ToList();
// Create binding and then bind input/ output features
_binding = new LearningModelBinding(_session);
_inputImageDescriptor =
inputFeatures.FirstOrDefault(feature => feature.Kind == LearningModelFeatureKind.Tensor) as TensorFeatureDescriptor;
_outputTensorDescriptor =
outputFeatures.FirstOrDefault(feature => feature.Kind == LearningModelFeatureKind.Tensor) as TensorFeatureDescriptor;
TensorFloat outputTensor = TensorFloat.Create(_outputTensorDescriptor.Shape);
ImageFeatureValue imageTensor = ImageFeatureValue.CreateFromVideoFrame(_inputFrame);
// Bind inputs +outputs
_binding.Bind(_inputImageDescriptor.Name, imageTensor);
_binding.Bind(_outputTensorDescriptor.Name, outputTensor);
// Evaluate and get the results
var results = await _session.EvaluateAsync(_binding, "test");
Debug.WriteLine("ResultsEvaluated: " + results.ToString());
var outputTensorList = outputTensor.GetAsVectorView();
var resultsList = new List <float>(outputTensorList.Count);
for (int i = 0; i < outputTensorList.Count; i++)
{
resultsList.Add(outputTensorList[i]);
}
}
catch (Exception ex)
{
Debug.WriteLine($"error: {ex.Message}");
_model = null;
}
}
public async Task <TensorFloat> EvaluateAsync(TensorFloat input)
{
binding.Bind(inName, input);
binding.Bind(outName, TensorFloat.Create(new long[] { 1, 1, InHeight, OutHeight }));
var result = await session.EvaluateAsync(binding, inName);
return(result.Outputs[outName] as TensorFloat);
}
public static async Task <TensorFloat> GetAsTensorFloat(this IRandomAccessStream stream,
int imageSize, float[] mean = null, float[] std = null)
{
BitmapDecoder decoder = await BitmapDecoder.CreateAsync(stream);
var softwareBitmap = await decoder.GetSoftwareBitmapAsync();
softwareBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Rgba8, BitmapAlphaMode.Premultiplied);
WriteableBitmap innerBitmap = new WriteableBitmap(softwareBitmap.PixelWidth, softwareBitmap.PixelHeight);
softwareBitmap.CopyToBuffer(innerBitmap.PixelBuffer);
int channelSize = imageSize * imageSize;
using (var context = innerBitmap.GetBitmapContext())
{
int[] src = context.Pixels;
var normalized = new float[imageSize * imageSize * 3];
for (var x = 0; x < imageSize; x++)
{
for (var y = 0; y < imageSize; y++)
{
var color = innerBitmap.GetPixel(y, x);
float r, g, b;
r = color.R;
g = color.G;
b = color.B;
if (mean != null && std != null)
{
r /= 255f;
g /= 255f;
b /= 255f;
r = (r - mean[0]) / std[0];
g = (g - mean[1]) / std[1];
b = (b - mean[2]) / std[2];
}
var indexChannelR = (x * imageSize) + y;
var indexChannelG = indexChannelR + channelSize;
var indexChannelB = indexChannelG + channelSize;
normalized[indexChannelR] = r;
normalized[indexChannelG] = g;
normalized[indexChannelB] = b;
}
}
return(TensorFloat.CreateFromArray(new List <long>()
{
1, 3, imageSize, imageSize
}, normalized));
}
}
public TensorFloat ConvertPixelsByteToTensor(byte[] imagePixels, BitmapPixelFormat bitmapPixelFormat)
{
if (imagePixels == null)
{
throw new NullReferenceException();
}
var pixelsWithAlpha = imagePixels;
List <float> reds = new List <float>();
List <float> greens = new List <float>();
List <float> blues = new List <float>();
for (int i = 0; i < pixelsWithAlpha.Length; ++i)
{
switch (i % 4)
{
case 0:
if (bitmapPixelFormat == BitmapPixelFormat.Rgba8)
{
reds.Add((float)pixelsWithAlpha[i] / 255);
}
else if (bitmapPixelFormat == BitmapPixelFormat.Bgra8)
{
blues.Add((float)pixelsWithAlpha[i] / 255);
}
break;
case 1:
greens.Add((float)pixelsWithAlpha[i] / 255);
break;
case 2:
if (bitmapPixelFormat == BitmapPixelFormat.Rgba8)
{
blues.Add((float)pixelsWithAlpha[i] / 255);
}
else if (bitmapPixelFormat == BitmapPixelFormat.Bgra8)
{
reds.Add((float)pixelsWithAlpha[i] / 255);
}
break;
}
}
List <float> sortedPixels = new List <float>();
sortedPixels.AddRange(reds);
sortedPixels.AddRange(greens);
sortedPixels.AddRange(blues);
long[] dimensions = { 1, 3, 416, 416 };
//Tensor<float> pixelTensor = new DenseTensor<float>(dimensions);
var inputTesnor = TensorFloat.CreateFromShapeArrayAndDataArray(dimensions, sortedPixels.ToArray());
//sortedPixels = null;
//sortedPixels.Clear();
//sortedPixels.TrimExcess();
return(inputTesnor);
}
private void SampleInputsGridView_SelectionChanged(object sender, SelectionChangedEventArgs e)
{
var gridView = sender as GridView;
var thumbnail = gridView.SelectedItem as WinMLSamplesGallery.Controls.Thumbnail;
if (thumbnail != null)
{
var image = thumbnail.ImageUri;
var file = StorageFile.GetFileFromApplicationUriAsync(new Uri(image)).GetAwaiter().GetResult();
var softwareBitmap = CreateSoftwareBitmapFromStorageFile(file);
tensorizationSession_ =
CreateLearningModelSession(
TensorizationModels.ReshapeFlatBufferNHWC(
1,
4,
softwareBitmap.PixelHeight,
softwareBitmap.PixelWidth,
416,
416));
// Tensorize
var stream = file.OpenAsync(FileAccessMode.Read).GetAwaiter().GetResult();
var decoder = BitmapDecoder.CreateAsync(stream).GetAwaiter().GetResult();
var bitmap = decoder.GetSoftwareBitmapAsync(BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied).GetAwaiter().GetResult();
var pixelDataProvider = decoder.GetPixelDataAsync().GetAwaiter().GetResult();
var bytes = pixelDataProvider.DetachPixelData();
var buffer = bytes.AsBuffer(); // Does this do a copy??
var inputRawTensor = TensorUInt8Bit.CreateFromBuffer(new long[] { 1, buffer.Length }, buffer);
// 3 channel NCHW
var tensorizeOutput = TensorFloat.Create(new long[] { 1, 416, 416, 3 });
var b = new LearningModelBinding(tensorizationSession_);
b.Bind(tensorizationSession_.Model.InputFeatures[0].Name, inputRawTensor);
b.Bind(tensorizationSession_.Model.OutputFeatures[0].Name, tensorizeOutput);
tensorizationSession_.Evaluate(b, "");
// Resize
var resizeBinding = new LearningModelBinding(_session);
resizeBinding.Bind(_session.Model.InputFeatures[0].Name, tensorizeOutput);
var results = _session.Evaluate(resizeBinding, "");
var output1 = results.Output(0) as TensorFloat;
var data = output1.GetAsVectorView();
var detections = ParseResult(data.ToList <float>().ToArray());
Comparer cp = new Comparer();
detections.Sort(cp);
var final = NMS(detections);
RenderImageInMainPanel(softwareBitmap);
}
}
public IEnumerable <float> EstimateDepth(float[] inTensor)
{
var task = Task.Run(async() =>
{
return(await onnxModel.EvaluateAsync(
TensorFloat.CreateFromArray(new long[] { 1, 3, InputHeight, InputWidth }, inTensor)));
});
return(task.Result.GetAsVectorView());
}
/// <summary>
/// PostProcessing.
/// Processes scors and boxes and generate a list of face rectangles.
/// </summary>
/// <param name="scores">score output of Onnx model.</param>
/// <param name="boxes">box output of Onnx model.</param>
public static IEnumerable <FaceDetectionRectangle> Predict(TensorFloat scores, TensorFloat boxes)
{
var boundingBoxCollection = new List <FaceDetectionRectangle>();
GenerateBBox(boundingBoxCollection, scores, boxes, _scoreThreshold);
var faceList = new List <FaceDetectionRectangle>();
NonMaximumSuppression(boundingBoxCollection, faceList, _iouThreshold);
return(faceList);
}
private async Task <List <float> > EvaluateFrame(VideoFrame frame)
{
_binding.Clear();
_binding.Bind("input_1:0", frame);
var results = await _session.EvaluateAsync(_binding, "");
TensorFloat result = results.Outputs["Identity:0"] as TensorFloat;
var shape = result.Shape;
var data = result.GetAsVectorView();
return(data.ToList <float>());
}
private async void PredictHangul(VideoFrame inputimage)
{
Stopwatch sw = new Stopwatch();
sw.Start();
// convert to bgr8
SoftwareBitmap bitgray8 = SoftwareBitmap.Convert(inputimage.SoftwareBitmap, BitmapPixelFormat.Gray8);
var buff = new byte[64 * 64];
bitgray8.CopyToBuffer(buff.AsBuffer());
var fbuff = new float[4096];
for (int i = 0; i < 4096; i++)
{
fbuff[i] = (float)buff[i] / 255;
}
long[] shape = { 1, 4096 };
charInput.input00 = TensorFloat.CreateFromArray(shape, fbuff);
var dummy = new float[1];
long[] dummy_shape = { };
charInput.keep_prob = TensorFloat.CreateFromArray(dummy_shape, dummy);
//Evaluate the model
charOuput = await charModel.EvaluateAsync(charInput);
//Convert output to datatype
IReadOnlyList <float> VectorImage = charOuput.output00.GetAsVectorView();
IList <float> ImageList = VectorImage.ToList();
//Display top results
var topPred = ImageList.Select((value, index) => new { index, value })
.ToDictionary(pair => pair.index, pair => pair.value)
.OrderByDescending(key => key.Value)
.ToArray();
string topLabeltxt = "";
for (int i = 1; i < 6; i++)
{
var item = topPred[i];
Debug.WriteLine($"{item.Key}, {item.Value}, {charLabel[item.Key]}");
topLabeltxt += $"{charLabel[item.Key]} ";
}
numberLabel.Text = charLabel[topPred[0].Key];
topLabel.Text = topLabeltxt;
Debug.WriteLine($"process time = {sw.Elapsed}");
}
private bool disposedValue = false; // 要检测冗余调用
void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
// TODO: 释放托管状态(托管对象)。
}
// TODO: 释放未托管的资源(未托管的对象)并在以下内容中替代终结器。
// TODO: 将大型字段设置为 null。
Grid = null;
disposedValue = true;
}
}
private async void Button_Click(object sender, RoutedEventArgs e)
{
// Use Picket to get file
var file = await GetImageFile();
SoftwareBitmap softwareBitmap;
byte[] bytes;
// Load image & scale to tensor input dimensions
using (IRandomAccessStream stream = await file.OpenAsync(FileAccessMode.Read))
{
bytes = await GetImageAsByteArrayAsync(stream, 320, 320, BitmapPixelFormat.Rgba8);
softwareBitmap = await GetImageAsSoftwareBitmapAsync(stream, 320, 320, BitmapPixelFormat.Bgra8);
}
// Display source image
var source = new SoftwareBitmapSource();
await source.SetBitmapAsync(softwareBitmap);
sourceImage.Source = source;
// Convert rgba-rgba-...-rgba to bb...b-rr...r-gg...g as colour weighted tensor (0..1)
TensorFloat input = TensorFloat.CreateFromIterable(new long[] { 1, 3, 320, 320 }, TensorBrg(bytes));
// Load model & perform inference
StorageFile modelFile = await StorageFile.GetFileFromApplicationUriAsync(new Uri($"ms-appx:///Assets/u2net.onnx"));
u2netModel model = await u2netModel.CreateFromStreamAsync(modelFile);
Stopwatch sw = new Stopwatch();
sw.Start();
u2netOutput output = await model.EvaluateAsync(new u2netInput { input = input });
sw.Stop();
await ToImage(output.o6, o6);
await ToImage(output.o5, o5);
await ToImage(output.o4, o4);
await ToImage(output.o3, o3);
await ToImage(output.o2, o2);
await ToImage(output.o1, o1);
await ToBlendedImage(bytes, output.o0, targetImage);
}
internal async Task <List <DetectionResult> > EvaluateFrame(VideoFrame frame)
{
_binding.Clear();
_binding.Bind("input_1:0", frame);
var results = await _session.EvaluateAsync(_binding, "");
TensorFloat result = results.Outputs["Identity:0"] as TensorFloat;
var shape = result.Shape;
var data = result.GetAsVectorView();
var detections = ParseResult(data.ToList <float>().ToArray());
Comparer cp = new Comparer();
detections.Sort(cp);
return(NMS(detections));
}
private static TensorFloat Normalize(byte[] src, System.Numerics.Vector3 mean, System.Numerics.Vector3 std, uint width, uint height)
{
var normalized = new float[src.Length / 4 * 3];
for (int i = 0; i < src.Length / 4; i++)
{
var val = src[i];
normalized[i * 3 + 0] = ((src[4 * i] / 255f) - mean.X) / std.X;
normalized[i * 3 + 1] = ((src[4 * i + 1] / 255f) - mean.Y) / std.Y;
normalized[i * 3 + 2] = ((src[4 * i + 2] / 255f) - mean.Z) / std.Z;
}
var shape = new List <long> {
3, width, height
};
return(TensorFloat.CreateFromArray(shape, normalized));
}
private async Task ToBlendedImage(byte[] data, TensorFloat tensorFloat, Image target)
{
var image = ApplyTensorAsMask(data, tensorFloat, 0.0f).ToArray();
var writeableBitmap = new WriteableBitmap(320, 320);
// Open a stream to copy the image contents to the WriteableBitmap's pixel buffer
using (Stream stream = writeableBitmap.PixelBuffer.AsStream())
{
await stream.WriteAsync(image, 0, image.Length);
}
var dest = SoftwareBitmap.CreateCopyFromBuffer(writeableBitmap.PixelBuffer, BitmapPixelFormat.Bgra8, 320, 320, BitmapAlphaMode.Premultiplied);
var destSouce = new SoftwareBitmapSource();
await destSouce.SetBitmapAsync(dest);
target.Source = destSouce;
}
internal String Evaluate()
{
// input tensor shape is [1x4]
long[] shape = new long[2];
shape[0] = 1;
shape[1] = 4;
// set up the input tensor
float[] input_data = new float[4];
input_data[0] = _sepal_length;
input_data[1] = _sepal_width;
input_data[2] = _petal_length;
input_data[3] = _petal_width;
TensorFloat tensor_float = TensorFloat.CreateFromArray(shape, input_data);
// bind the tensor to "input"
var binding = new LearningModelBinding(_session);
binding.Bind("input", tensor_float);
// evaluate
var results = _session.Evaluate(binding, "");
// get the results
TensorFloat prediction = (TensorFloat)results.Outputs.First().Value;
var prediction_data = prediction.GetAsVectorView();
// find the highest predicted value
int max_index = 0;
float max_value = 0;
for (int i = 0; i < prediction_data.Count; i++)
{
var val = prediction_data.ElementAt(i);
if (val > max_value)
{
max_value = val;
max_index = i;
}
}
// return the label corresponding to the highest predicted value
return(_labels.ElementAt(max_index));
}
/// <summary>
/// Processes scors and boxes and generate a list of face rectangles.
/// </summary>
/// <param name="landmarkTensors">landmark output of Onnx model.</param>
/// <param name="imageX">X start position of the image.</param>
/// <param name="imageY">Y start position of the image.</param>
/// <param name="imageWidth">width of the image.</param>
/// <param name="imageHeight">height of the image.</param>
public static FaceLandmarks Predict(TensorFloat landmarkTensors, int imageX, int imageY, int imageWidth, int imageHeight)
{
var faceLandmarks = new FaceLandmarks();
IReadOnlyList <float> vectorLandmarks = landmarkTensors.GetAsVectorView();
IList <float> landmarkFloatList = vectorLandmarks.ToList();
long numAnchors = (long)Math.Ceiling(landmarkTensors.Shape[1] * 0.5);
for (var i = 0; i < numAnchors; i++)
{
var mark = new FaceLandmark
{
X = landmarkFloatList[i * 2] * imageWidth + imageX,
Y = landmarkFloatList[i * 2 + 1] * imageHeight + imageY
};
faceLandmarks.landmarkList.Add(mark);
}
return(faceLandmarks);
}
private async void Button_Click(object sender, RoutedEventArgs e)
{
var shape = new long[] { 1, 1 };
var modelInput = new taxiFarePredInput()
{
PassengerCount = TensorFloat.CreateFromArray(shape, new float[] { 1f }),
TripTime = TensorFloat.CreateFromArray(shape, new float[] { 1140f }),
TripDistance = TensorFloat.CreateFromArray(shape, new float[] { 3.75f }),
FareAmount = TensorFloat.CreateFromArray(shape, new float[] { 0f }),
};
var modelFile = await StorageFile.GetFileFromApplicationUriAsync(new Uri("ms-appx:///Assets/taxiFarePred.onnx"));
var session = await taxiFarePredModel.CreateFromStreamAsync(modelFile);
var modelOutput = await session.EvaluateAsync(modelInput);
var score = modelOutput.Score0.GetAsVectorView();
btn1.Content = score[0];
}
public override List <ONNXTensor> Run(IEnumerable <string> outputs, Dictionary <string, ONNXTensor> feedDict)
{
var binding = new LearningModelBinding(sess);
foreach (var item in feedDict)
{
object tensor;
if (!IsFP16)
{
tensor = TensorFloat.CreateFromArray(item.Value.Shape, item.Value.Buffer);
if (IsGPU)
{
//TODO: Move SoftwareTensor to DX12Tensor
tensor = MoveToGPU((TensorFloat)tensor);
}
}
else
{
tensor = TensorFloat16Bit.CreateFromArray(item.Value.Shape, item.Value.Buffer);
}
binding.Bind(item.Key, tensor);
}
var result = sess.Evaluate(binding, $"eval{++evalCount}");
var ret = new List <ONNXTensor>();
foreach (var item in outputs)
{
var tensor = result.Outputs[item] as TensorFloat;
var vector = tensor.GetAsVectorView().ToArray();
ret.Add(new ONNXTensor()
{
Buffer = vector, Shape = tensor.Shape.ToArray()
});
}
return(ret);
}
private async Task <List <float> > ArcFace(SoftwareBitmap softwareBitmap)
{
// Encapsulate the image within a VideoFrame to be bound and evaluated
VideoFrame inputImage = VideoFrame.CreateWithSoftwareBitmap(softwareBitmap);
int height = inputImage.SoftwareBitmap.PixelHeight;
int width = inputImage.SoftwareBitmap.PixelWidth;
float[] data = new float[1 * 3 * ARC_FACE_INPUT * ARC_FACE_INPUT];
byte[] imageBytes = new byte[4 * height * width];
inputImage.SoftwareBitmap.CopyToBuffer(imageBytes.AsBuffer());
int id = 0;
for (int i = 0; i < data.Length; i += 4)
{
float blue = (float)imageBytes[i];
float green = (float)imageBytes[i + 1];
float red = (float)imageBytes[i + 2];
data[id++] = blue;
data[id++] = green;
data[id++] = red;
}
_arcFaceInput.data = TensorFloat.CreateFromArray(new List <long> {
1, 3, ARC_FACE_INPUT, ARC_FACE_INPUT
}, data);
// Process the frame with the model
_arcFaceOutput = await _arcFaceModel.EvaluateAsync(_arcFaceInput);
IReadOnlyList <float> vectorImage = _arcFaceOutput.fc1.GetAsVectorView();
return(vectorImage.ToList());
}
private static LearningModelEvaluationResult Evaluate(LearningModelSession session, object input)
{
// Create the binding
var binding = new LearningModelBinding(session);
// Create an empty output, that will keep the output resources on the GPU
// It will be chained into a the post processing on the GPU as well
var output = TensorFloat.Create();
// Bind inputs and outputs
// For squeezenet these evaluate to "data", and "squeezenet0_flatten0_reshape0"
string inputName = session.Model.InputFeatures[0].Name;
string outputName = session.Model.OutputFeatures[0].Name;
binding.Bind(inputName, input);
var outputBindProperties = new PropertySet();
outputBindProperties.Add("DisableTensorCpuSync", PropertyValue.CreateBoolean(true));
binding.Bind(outputName, output, outputBindProperties);
// Evaluate
return(session.Evaluate(binding, ""));
}
/// <summary>
/// Evaluate the VideoFrame passed in as arg
/// </summary>
/// <param name="inputFrame"></param>
/// <returns></returns>
private async Task EvaluateVideoFrameAsync(VideoFrame inputFrame)
{
if (inputFrame != null)
{
try
{
StatusBlock.Text = "Binding image...";
// create a binding object from the session
LearningModelBinding binding = new LearningModelBinding(_session);
// bind the input image
ImageFeatureValue imageTensor = ImageFeatureValue.CreateFromVideoFrame(inputFrame);
binding.Bind("data_0", imageTensor);
// temp: there is a bug where winml doesn't allow unbound outputs yet, prebind the output!
{
TensorFeatureDescriptor outputTensorDescription = _model.OutputFeatures.FirstOrDefault(
feature => feature.Name == "softmaxout_1"
) as TensorFeatureDescriptor;
TensorFloat outputTensor = TensorFloat.Create(outputTensorDescription.Shape);
binding.Bind("softmaxout_1", outputTensor);
}
StatusBlock.Text = "Running model...";
int ticks = Environment.TickCount;
// Process the frame with the model
var results = await _session.EvaluateAsync(binding, $"Run { ++_runCount } ");
ticks = Environment.TickCount - ticks;
// retrieve results from evaluation
var resultTensor = results.Outputs["softmaxout_1"] as TensorFloat;
var resultVector = resultTensor.GetAsVectorView();
// Find the top 3 probabilities
List <float> topProbabilities = new List <float>()
{
0.0f, 0.0f, 0.0f
};
List <int> topProbabilityLabelIndexes = new List <int>()
{
0, 0, 0
};
// SqueezeNet returns a list of 1000 options, with probabilities for each, loop through all
for (int i = 0; i < resultVector.Count(); i++)
{
// is it one of the top 3?
for (int j = 0; j < 3; j++)
{
if (resultVector[i] > topProbabilities[j])
{
topProbabilityLabelIndexes[j] = i;
topProbabilities[j] = resultVector[i];
break;
}
}
}
// Display the result
string message = $"Run took { ticks } ticks";
for (int i = 0; i < 3; i++)
{
message += $"\n\"{ _labels[topProbabilityLabelIndexes[i]]}\" with confidence of { topProbabilities[i]}";
}
StatusBlock.Text = message;
}
catch (Exception ex)
{
StatusBlock.Text = $"error: {ex.Message}";
}
ButtonRun.IsEnabled = true;
}
}
static async Task <int> Main(string[] args)
{
try
{
//
// Parse options
//
Options = new AppOptions();
Options.Parse(args);
if (Options.ShowList)
{
}
if (Options.Exit)
{
return(-1);
}
if (string.IsNullOrEmpty(Options.FileName))
{
throw new ApplicationException("Please use --file to specify which file to use");
}
//
// Init module client
//
if (Options.UseEdge)
{
Log.WriteLine($"{AppOptions.AppName} module starting.");
await BlockTimer("Initializing Azure IoT Edge", async() => await InitEdge());
}
cts = new CancellationTokenSource();
AssemblyLoadContext.Default.Unloading += (ctx) => cts.Cancel();
Console.CancelKeyPress += (sender, cpe) => cts.Cancel();
//
// Load model
//
MLModel model = null;
Console.WriteLine($"Loading model from: '{Options.ModelPath}', Exists: '{File.Exists(Options.ModelPath)}'");
await BlockTimer($"Loading modelfile '{Options.ModelPath}' on the {(Options.UseGpu ? "GPU" : "CPU")}",
async() =>
{
var d = Directory.GetCurrentDirectory();
var path = d + "\\" + Options.ModelPath;
StorageFile modelFile = await AsAsync(StorageFile.GetFileFromPathAsync(path));
model = await MLModel.CreateFromStreamAsync(modelFile);
});
do
{
//
// Open file
//
var rows = new List <DataRow>();
try
{
using (var fs = new StreamReader(Options.FileName))
{
// I just need this one line to load the records from the file in my List<CsvLine>
rows = new CsvHelper.CsvReader(fs).GetRecords <DataRow>().ToList();
Console.WriteLine($"Loaded {rows.Count} row(s)");
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
Console.WriteLine(rows);
//
// Main loop
//
foreach (var row in rows)
{
//
// Evaluate model
//
var inputShape = new long[2] {
1, 4
};
var inputFeatures = new float[4] {
row.Temperature, row.Pressure, row.Humidity, row.ExternalTemperature
};
MLModelVariable result = null;
var evalticks = await BlockTimer("Running the model",
async() =>
{
result = await model.EvaluateAsync(new MLModelVariable()
{
Variable = TensorFloat.CreateFromArray(inputShape, inputFeatures)
});
});
//
// Print results
//
var message = new MessageBody
{
result = result.Variable.GetAsVectorView().First()
};
message.metrics.evaltimeinms = evalticks;
var json = JsonConvert.SerializeObject(message);
Log.WriteLineRaw($"Recognized {json}");
//
// Send results to Edge
//
if (Options.UseEdge)
{
var eventMessage = new Message(Encoding.UTF8.GetBytes(json));
await ioTHubModuleClient.SendEventAsync("resultsOutput", eventMessage);
// Let's not totally spam Edge :)
await Task.Delay(500);
}
Console.WriteLine("Waiting 1 second...");
Thread.Sleep(1000);
}
}while (Options.RunForever && !cts.Token.IsCancellationRequested);
}
catch (Exception ex)
{
Console.WriteLine(ex);
return(-1);
}
return(0);
}
