public hate_coremlOutput(MLMultiArray output1)
{
if (output1 == null)
{
throw new ArgumentNullException(nameof(output1));
}
Output1 = output1;
}
public FreeSoundsPlusModel25Input(MLMultiArray audioSamples)
{
if (audioSamples == null)
{
throw new ArgumentNullException(nameof(audioSamples));
}
AudioSamples = audioSamples;
}
public hate_coremlInput(MLMultiArray input1)
{
if (input1 == null)
{
throw new ArgumentNullException(nameof(input1));
}
Input1 = input1;
}
/// <summary>
/// Checks if the specified <see cref="MLMultiArray"/> has correct values at (<paramref name="i"/>,
/// <paramref name="j"/>) of the chip.
/// </summary>
/// <param name="pixel">The specified 2D array of <see cref="SKColor"/>s.</param>
/// <param name="result">The specified <see cref="MLMultiArray"/>.</param>
/// <param name="i">The specified index i.</param>
/// <param name="j">The specified index j.</param>
private static void AssertMLMultiArrayEntry(SKColor pixel, MLMultiArray result, int i, int j)
{
var h = SharedConstants.DefaultChipHeight - j - 1;
var w = SharedConstants.DefaultChipWidth - i - 1;
Assert.AreEqual(ColorMapping[pixel.Red].DoubleValue,
result[PreProcessingService.ComputeIndex(0, h, w)].DoubleValue, Tolerance);
Assert.AreEqual(ColorMapping[pixel.Green].DoubleValue,
result[PreProcessingService.ComputeIndex(1, h, w)].DoubleValue, Tolerance);
Assert.AreEqual(ColorMapping[pixel.Blue].DoubleValue,
result[PreProcessingService.ComputeIndex(2, h, w)].DoubleValue, Tolerance);
}
/// <summary>
/// Runs the input through the CoreML model
/// </summary>
/// <param name="input">The input data to the model.</param>
/// <returns>The output of the CoreML model.</returns>
private MLMultiArray Predict(MLMultiArray input)
{
var inputFeatureProvider = new ImageInputFeatureProvider(inputFeatureName)
{
ImagePixelData = input
};
MLMultiArray result = this.mlModel.GetPrediction(inputFeatureProvider, out NSError modelErr)
.GetFeatureValue(outputFeatureName)
.MultiArrayValue;
return(result);
}
/// <summary>
/// Extract the direct output from MLCore without post-processing
/// </summary>
/// <returns>A tuple contain the two matrices output by the model. First matrix: the box prediction encoding; second matrix: the class logit for each box.</returns>
/// <param name="input"> The formatted matrix representation of image after proper transpose and normalization</param>
private (MLMultiArray, MLMultiArray) RawPredict(MLMultiArray input)
{
var inputFeatureProvider = new ImageInputFeatureProvider(_inputFeatureName)
{
ImagePixelData = input
};
var resultBundle = _mlModel.GetPrediction(inputFeatureProvider, out _);
var output1 = resultBundle.GetFeatureValue(Constants.ModelOutputBoxFeatureName).MultiArrayValue;
var output2 = resultBundle.GetFeatureValue(Constants.ModelOutputClassFeatureName).MultiArrayValue;
inputFeatureProvider.Dispose();
return(output1, output2);
}
/// <summary>
/// Implements <see cref="IAIImageModel.Predict(SKBitmap)"/> using a CoreML model
/// </summary>
public float[] Predict(SKBitmap imageData)
{
MLMultiArray input = this.ExtractPixelData(imageData);
MLMultiArray output = this.Predict(input);
// Transfer the results from a MLMultiArray to a regular float[]
float[] outputFloats = new float[output.Count];
for (int i = 0; i < output.Count; i++)
{
outputFloats[i] = (float)output[i];
}
return(outputFloats);
}
public ObjectDetectorOutput(MLMultiArray confidence, MLMultiArray coordinates)
{
if (confidence == null)
{
throw new ArgumentNullException(nameof(confidence));
}
if (coordinates == null)
{
throw new ArgumentNullException(nameof(coordinates));
}
Confidence = confidence;
Coordinates = coordinates;
}
/// <summary>
/// Converts the pixels from the specified <see cref="Chip"/> to a <see cref="MLMultiArray"/>.
/// </summary>
/// <returns>The <see cref="MLMultiArray"/> corresponding to the pixels.</returns>
/// <param name="chip">The <see cref="Chip"/> to convert.</param>
public static MLMultiArray ToMLMultiArray(this Chip chip)
{
if (chip == null)
{
throw new ArgumentNullException();
}
// mlMultiArray is transposed with respect to the diagonal
var mlMultiArray =
new MLMultiArray(
new NSNumber[]
{
Constants.SequenceLength,
Constants.BatchSize,
Constants.ImageChannelSize,
SharedConstants.DefaultChipHeight,
SharedConstants.DefaultChipWidth
},
MLMultiArrayDataType.Double,
out _);
// Currently we assume Constants.SequenceLength == 1 and Constants.BatchSize == 1.
// Once we support sequence and batch input, we should refactor this and the ComputeIndex method.
for (var i = 0; i < SharedConstants.DefaultChipWidth; i++)
{
for (var j = 0; j < SharedConstants.DefaultChipHeight; j++)
{
var h = SharedConstants.DefaultChipHeight - j - 1;
var w = SharedConstants.DefaultChipWidth - i - 1;
if (i < chip.Region.Width && j < chip.Region.Height)
{
mlMultiArray[ComputeIndex(0, h, w)] = ColorMapping[chip.Pixels[i][j].Red];
mlMultiArray[ComputeIndex(1, h, w)] = ColorMapping[chip.Pixels[i][j].Green];
mlMultiArray[ComputeIndex(2, h, w)] = ColorMapping[chip.Pixels[i][j].Blue];
}
else
{
mlMultiArray[ComputeIndex(0, h, w)] = ColorMapping[0];
mlMultiArray[ComputeIndex(1, h, w)] = ColorMapping[0];
mlMultiArray[ComputeIndex(2, h, w)] = ColorMapping[0];
}
}
}
return(mlMultiArray);
}
public void Ctors()
{
TestRuntime.AssertXcodeVersion(9, 0);
NSError err;
var shape = new nint [] { 1 };
var strides = new nint [] { 0 };
var nsshape = new NSNumber [] { NSNumber.FromNInt(1) };
var nsstrides = new NSNumber [] { NSNumber.FromNInt(0) };
using (var arr = new MLMultiArray(shape, MLMultiArrayDataType.Int32, out err)) {
Assert.AreEqual(shape, arr.Shape, "1 Shape");
Assert.AreEqual(MLMultiArrayDataType.Int32, arr.DataType, "1 DataType");
Assert.IsNull(err, "1 err");
}
using (var arr = new MLMultiArray(IntPtr.Zero, shape, MLMultiArrayDataType.Float32, strides, (v) => Marshal.FreeHGlobal(v), out err)) {
Assert.AreEqual(shape, arr.Shape, "2 Shape");
Assert.AreEqual(MLMultiArrayDataType.Float32, arr.DataType, "2 DataType");
Assert.AreEqual(strides, arr.Strides, "2 Strides");
Assert.IsNull(err, "2 err");
}
using (var arr = new MLMultiArray(IntPtr.Zero, nsshape, MLMultiArrayDataType.Double, nsstrides, (v) => Marshal.FreeHGlobal(v), out err)) {
Assert.AreEqual(shape, arr.Shape, "3 Shape");
Assert.AreEqual(MLMultiArrayDataType.Double, arr.DataType, "3 DataType");
Assert.AreEqual(strides, arr.Strides, "3 Strides");
Assert.AreEqual(IntPtr.Zero, arr.DataPointer, "3 DataPointer");
Assert.IsNull(err, "3 err");
}
using (var arr = new MLMultiArray(nsshape, MLMultiArrayDataType.Int32, out err)) {
Assert.AreEqual(shape, arr.Shape, "4 Shape");
Assert.AreEqual(MLMultiArrayDataType.Int32, arr.DataType, "4 DataType");
Assert.IsNull(err, "4 err");
}
}
/// <summary>
/// Make a prediction using the convenience interface
/// </summary>
/// <param name="audioSamples">Input audio samples to be classified as 15600 1-dimensional array of floats</param>
/// <param name="options">prediction options</param>
/// <param name="error">If an error occurs, upon return contains an NSError object that describes the problem.</param>
public FreeSoundsPlusModel25Output GetPrediction(MLMultiArray audioSamples, MLPredictionOptions options, out NSError error)
{
var input = new FreeSoundsPlusModel25Input(audioSamples);
return(GetPrediction(input, options, out error));
}
/// <summary>
/// Make a prediction using the convenience interface
/// </summary>
/// <param name="input1"> as 1 1-dimensional array of doubles</param>
/// <param name="options">prediction options</param>
/// <param name="error">If an error occurs, upon return contains an NSError object that describes the problem.</param>
public hate_coremlOutput GetPrediction(MLMultiArray input1, MLPredictionOptions options, out NSError error)
{
var input = new hate_coremlInput(input1);
return(GetPrediction(input, options, out error));
}
/// <summary>
/// Runs CoreML on the image in SKBitmap format .
/// </summary>
/// <returns>A <see cref="CategoryManager"/> that contains all observations detected from the specified
/// <see cref="MLMultiArray"/>.</returns>
/// <param name="input">The array representation of image after pre-processing </param>
public CategoryManager Predict(MLMultiArray input)
{
MLMultiArray output1;
MLMultiArray output2;
(output1, output2) = RawPredict(input);
// checking for expected dimensions
if (output1.Shape == new nint[] { 4, NumAnchorBox, 1 })
{
// one anchoring box is represented by 4 points, no duplicating anchoring box
return(null);
}
if (output2.Shape == new nint[] { 1, 1, NumClasses, 1, NumAnchorBox })
{
return(null);
}
// conversion from MLMultiArray to double readable content
var outputFloats1 = new double[output1.Count]; // box prediction encoding 4 x 1917 x 1
for (var i = 0; i < output1.Count; i++)
{
outputFloats1[i] = (double)output1[i];
}
var outputFloats2 = new double[output2.Count]; // confidence score output matrix : 1 x 1 x 23 x 1 x 1917
for (var i = 0; i < output2.Count; i++)
{
outputFloats2[i] = (double)output2[i];
}
output1.Dispose();
output2.Dispose();
var cm = new CategoryManager();
for (var ibox = 0; ibox < NumAnchorBox; ibox++)
{
var scoresOfClasses = new List <double>();
for (var iclass = 0; iclass < NumClasses; iclass++)
{
var offsetIndex = iclass * NumAnchorBox + ibox;
var logit = outputFloats2[offsetIndex];
scoresOfClasses.Add(logit);
}
scoresOfClasses = MachineLearningService.Softmax(scoresOfClasses).ToList();
for (var iclass = 0; iclass < NumClasses; iclass++)
{
var confidence = scoresOfClasses[iclass];
var box = GenerateSKRect(
outputFloats1[BoxIndexOffset(0, ibox)],
outputFloats1[BoxIndexOffset(1, ibox)],
outputFloats1[BoxIndexOffset(2, ibox)],
outputFloats1[BoxIndexOffset(3, ibox)],
ibox);
if (iclass != 0 && IsWithinFrame(box))
{
var _ = new Observation(cm.GetOrCreate(iclass, ModelClassLabelEncoding.LookUpLabel(iclass)),
confidence, box);
}
}
}
return(cm);
}
public void Indexers()
{
TestRuntime.AssertXcodeVersion(9, 0);
NSError err;
var shape = new nint [] { 10 };
using (var arr = new MLMultiArray(shape, MLMultiArrayDataType.Int32, out err)) {
Assert.IsNull(err, "err");
Assert.AreEqual(10, arr.Count, "Count");
Assert.AreEqual(new nint [] { 10 }, arr.Shape, "Shape");
Assert.AreEqual(new nint [] { 1 }, arr.Strides, "Strides");
arr [0] = 0; // MLMultiArray's elements aren't zero-initialized
Assert.AreEqual(0, arr [0].Int32Value, "a");
Assert.AreEqual(0, arr [new nint [] { 0 }].Int32Value, "b");
Assert.AreEqual(0, arr [new NSNumber [] { NSNumber.FromNInt(0) }].Int32Value, "c nint");
Assert.AreEqual(0, arr [new NSNumber [] { NSNumber.FromInt32(0) }].Int32Value, "c int32");
Assert.AreEqual(0, arr [new NSNumber [] { NSNumber.FromByte(0) }].Int32Value, "c byte");
Assert.AreEqual(0, arr [new NSNumber [] { NSNumber.FromFloat(0) }].Int32Value, "c float");
Assert.AreEqual(0, arr.GetObject(0).Int32Value, "GetObject a");
Assert.AreEqual(0, arr.GetObject(new nint [] { 0 }).Int32Value, "GetObject b");
Assert.AreEqual(0, arr.GetObject(new NSNumber [] { NSNumber.FromNInt(0) }).Int32Value, "GetObject c nint");
Assert.AreEqual(0, arr.GetObject(new NSNumber [] { NSNumber.FromInt32(0) }).Int32Value, "GetObject c int32");
Assert.AreEqual(0, arr.GetObject(new NSNumber [] { NSNumber.FromByte(0) }).Int32Value, "GetObject c byte");
Assert.AreEqual(0, arr.GetObject(new NSNumber [] { NSNumber.FromFloat(0) }).Int32Value, "GetObject c float");
arr [1] = NSNumber.FromInt32(1);
arr [new nint [] { 2 }] = NSNumber.FromInt32(2);
arr [new NSNumber [] { NSNumber.FromUInt16(3) }] = NSNumber.FromInt32(3);
arr.SetObject(NSNumber.FromInt32(4), 4);
arr.SetObject(NSNumber.FromInt32(5), new nint [] { 5 });
arr.SetObject(NSNumber.FromInt32(6), new NSNumber [] { NSNumber.FromSByte(6) });
Assert.AreEqual(1, arr [1].Int32Value, "1");
Assert.AreEqual(2, arr [2].Int32Value, "2");
Assert.AreEqual(3, arr [3].Int32Value, "3");
Assert.AreEqual(4, arr [4].Int32Value, "4");
Assert.AreEqual(5, arr [5].Int32Value, "5");
Assert.AreEqual(6, arr [6].Int32Value, "6");
}
// multi-dimensional
shape = new nint [] { 7, 7, 7 };
using (var arr = new MLMultiArray(shape, MLMultiArrayDataType.Int32, out err)) {
Assert.IsNull(err, "err");
Assert.AreEqual(shape [0] * shape [1] * shape [2], arr.Count, "Count");
arr [0, 0, 0] = 0; // MLMultiArray's elements aren't zero-initialized
Assert.AreEqual(0, arr [0, 0, 0].Int32Value, "a");
Assert.AreEqual(0, arr [new nint [] { 0, 0, 0 }].Int32Value, "b");
Assert.AreEqual(0, arr [new NSNumber [] { NSNumber.FromNInt(0), NSNumber.FromNInt(0), NSNumber.FromNInt(0) }].Int32Value, "c nint");
Assert.AreEqual(0, arr.GetObject(0, 0, 0).Int32Value, "GetObject a");
Assert.AreEqual(0, arr.GetObject(new nint [] { 0, 0, 0 }).Int32Value, "GetObject b");
Assert.AreEqual(0, arr.GetObject(new NSNumber [] { NSNumber.FromNInt(0), NSNumber.FromNInt(0), NSNumber.FromNInt(0) }).Int32Value, "GetObject c nint");
arr [1, 1, 1] = NSNumber.FromInt32(1);
arr [new nint [] { 2, 2, 2 }] = NSNumber.FromInt32(2);
arr [new NSNumber [] { NSNumber.FromUInt16(3), NSNumber.FromUInt16(3), NSNumber.FromUInt16(3) }] = NSNumber.FromInt32(3);
arr.SetObject(NSNumber.FromInt32(4), 4, 4, 4);
arr.SetObject(NSNumber.FromInt32(5), new nint [] { 5, 5, 5 });
arr.SetObject(NSNumber.FromInt32(6), new NSNumber [] { NSNumber.FromSByte(6), NSNumber.FromSByte(6), NSNumber.FromSByte(6) });
Assert.AreEqual(1, arr [1, 1, 1].Int32Value, "1");
Assert.AreEqual(2, arr [2, 2, 2].Int32Value, "2");
Assert.AreEqual(3, arr [3, 3, 3].Int32Value, "3");
Assert.AreEqual(4, arr [4, 4, 4].Int32Value, "4");
Assert.AreEqual(5, arr [5, 5, 5].Int32Value, "5");
Assert.AreEqual(6, arr [6, 6, 6].Int32Value, "6");
}
}
public List <string> PredictNextWord(string word)
{
NSError err = new NSError();
var input = new MLMultiArray(new nint[] { 1, 1 }, MLMultiArrayDataType.Int32, out err);
Dictionary <int, String> idx2word;
Dictionary <String, int> word2idx;
switch (Language)
{
case "pt":
if (model_pt == null)
{
model_pt = new predictor_pt();
}
idx2word = Newtonsoft.Json.JsonConvert.DeserializeObject <Dictionary <int, String> >(File.ReadAllText("idx2word_pt.txt"));
word2idx = Newtonsoft.Json.JsonConvert.DeserializeObject <Dictionary <String, int> >(File.ReadAllText("word2idx_pt.txt"));
if (!word2idx.ContainsKey(word))
{
return(new List <string>());
}
input[0] = word2idx[word];
var output = model_pt.GetPrediction(input, out err);
List <Double> res = new List <double>();
for (var i = 0; i < idx2word.Count + 1; i++)
{
res.Add(output.Identity[new nint[] { 0, 0, i }].DoubleValue);
}
return(res.OrderByDescending(x => x).Select(x2 => idx2word[res.IndexOf(x2)]).Take(10).ToList());
case "es":
if (model_es == null)
{
model_es = new predictor_es();
}
idx2word = Newtonsoft.Json.JsonConvert.DeserializeObject <Dictionary <int, String> >(File.ReadAllText("idx2word_es.txt"));
word2idx = Newtonsoft.Json.JsonConvert.DeserializeObject <Dictionary <String, int> >(File.ReadAllText("word2idx_es.txt"));
if (!word2idx.ContainsKey(word))
{
return(new List <string>());
}
input[0] = word2idx[word];
var output2 = model_es.GetPrediction(input, out err);
List <Double> res2 = new List <double>();
for (var i = 0; i < idx2word.Count + 1; i++)
{
res2.Add(output2.Identity[new nint[] { 0, 0, i }].DoubleValue);
}
return(res2.OrderByDescending(x => x).Select(x2 => idx2word[res2.IndexOf(x2)]).Take(10).ToList());
default:
if (model_en == null)
{
model_en = new predictor_en();
}
idx2word = Newtonsoft.Json.JsonConvert.DeserializeObject <Dictionary <int, String> >(File.ReadAllText("idx2word_en.txt"));
word2idx = Newtonsoft.Json.JsonConvert.DeserializeObject <Dictionary <String, int> >(File.ReadAllText("word2idx_en.txt"));
if (!word2idx.ContainsKey(word))
{
return(new List <string>());
}
input[0] = word2idx[word];
var output3 = model_en.GetPrediction(input, out err);
List <Double> res3 = new List <double>();
for (var i = 0; i < idx2word.Count + 1; i++)
{
res3.Add(output3.Identity[new nint[] { 0, 0, i }].DoubleValue);
}
return(res3.OrderByDescending(x => x).Select(x2 => idx2word[res3.IndexOf(x2)]).Take(10).ToList());
}
}
