public void MakePrediction(ARFaceAnchor faceAnchor)
{
if (testingDot == null)
{
return;
}
var gaze = GazePredictionInput.FromAnchor(faceAnchor, motionManager.DeviceMotion.Attitude);
try
{
var yPrediction = yModel.GetPrediction(gaze, out NSError yPredictionError);
var yResult = yPrediction.GetFeatureValue("yPoint").DoubleValue;
var xPrediction = xModel.GetPrediction(gaze, out NSError xPredictionError);
var xResult = xPrediction.GetFeatureValue("xPoint").DoubleValue;
System.Console.WriteLine($"Prediction is: ({xResult}, {yResult}");
testingDot.Position = new CGPoint(xResult, yResult);
}
catch (Exception ex)
{
System.Console.WriteLine(ex);
}
}
/// <summary>
/// Make a prediction using the standard interface
/// </summary>
/// <param name="input">an instance of MarsHabitatPricerInput to predict from</param>
/// <param name="error">If an error occurs, upon return contains an NSError object that describes the problem.</param>
public MarsHabitatPricerOutput GetPrediction(MarsHabitatPricerInput input, out NSError error)
{
if (input == null)
{
throw new ArgumentNullException(nameof(input));
}
var prediction = model.GetPrediction(input, out error);
if (prediction == null)
{
return(null);
}
var priceValue = prediction.GetFeatureValue("price").DoubleValue;
return(new MarsHabitatPricerOutput(priceValue));
}
/// <summary>
/// Make a prediction using the standard interface
/// </summary>
/// <param name="input">an instance of hate_coremlInput to predict from</param>
/// <param name="error">If an error occurs, upon return contains an NSError object that describes the problem.</param>
public hate_coremlOutput GetPrediction(hate_coremlInput input, out NSError error)
{
if (input == null)
{
throw new ArgumentNullException(nameof(input));
}
var prediction = model.GetPrediction(input, out error);
if (prediction == null)
{
return(null);
}
var output1Value = prediction.GetFeatureValue("output1").MultiArrayValue;
return(new hate_coremlOutput(output1Value));
}
/// <summary>
/// Make a prediction using the standard interface
/// </summary>
/// <param name="input">an instance of CookHappyJuneInput to predict from</param>
/// <param name="error">If an error occurs, upon return contains an NSError object that describes the problem.</param>
public CookHappyJuneOutput GetPrediction(CookHappyJuneInput input, out NSError error)
{
if (input == null)
{
throw new ArgumentNullException(nameof(input));
}
var prediction = model.GetPrediction(input, out error);
if (prediction == null)
{
return(null);
}
var lossValue = prediction.GetFeatureValue("loss").DictionaryValue;
var classLabelValue = prediction.GetFeatureValue("classLabel").StringValue;
return(new CookHappyJuneOutput(lossValue, classLabelValue));
}
/// <summary>
/// Make a prediction using the standard interface
/// </summary>
/// <param name="input">an instance of FreeSoundsPlusModel25Input to predict from</param>
/// <param name="error">If an error occurs, upon return contains an NSError object that describes the problem.</param>
public FreeSoundsPlusModel25Output GetPrediction(FreeSoundsPlusModel25Input input, out NSError error)
{
if (input == null)
{
throw new ArgumentNullException(nameof(input));
}
var prediction = model.GetPrediction(input, out error);
if (prediction == null)
{
return(null);
}
var classLabelProbsValue = prediction.GetFeatureValue("classLabelProbs").DictionaryValue;
var classLabelValue = prediction.GetFeatureValue("classLabel").StringValue;
return(new FreeSoundsPlusModel25Output(classLabelProbsValue, classLabelValue));
}
private List <Prediction> Classify(MLModel model, UIImage source)
{
if (model == null)
{
throw new ArgumentNullException(nameof(model));
}
if (source == null)
{
throw new ArgumentNullException(nameof(source));
}
var scaled = source.Scale(new CGSize(227, 227));
var pixelBuffer = ToCVPixelBuffer(scaled);
var imageValue = MLFeatureValue.Create(pixelBuffer);
var inputs = new NSDictionary <NSString, NSObject>(new NSString("data"), imageValue);
NSError error, error2;
var inputFp = new MLDictionaryFeatureProvider(inputs, out error);
if (error != null)
{
System.Diagnostics.Debug.WriteLine(error);
throw new InvalidOperationException(error.Description);
}
var outFeatures = model.GetPrediction(inputFp, out error2);
if (error2 != null)
{
System.Diagnostics.Debug.WriteLine(error2);
throw new InvalidOperationException(error2.Description);
}
var predictionsDictionary = outFeatures.GetFeatureValue("loss").DictionaryValue;
var byProbability = new List <Tuple <double, string> >();
foreach (var key in predictionsDictionary.Keys)
{
var description = (string)(NSString)key;
var prob = (double)predictionsDictionary[key];
byProbability.Add(new Tuple <double, string>(prob, description));
}
byProbability.Sort((t1, t2) => t1.Item1.CompareTo(t2.Item1) * -1);
var result = byProbability
.Select(p => new Prediction {
Label = p.Item2, Probability = p.Item1
})
.ToList();
return(result);
}
/// <summary>
/// Make a prediction using the standard interface
/// </summary>
/// <param name="input">an instance of MarsHabitatPricerInput to predict from</param>
/// <param name="error">If an error occurs, upon return contains an NSError object that describes the problem.</param>
public MarsHabitatPricerOutput GetPrediction(MarsHabitatPricerInput input, out NSError error)
{
var prediction = model.GetPrediction(input, out error);
if (prediction == null)
{
return(null);
}
var priceValue = prediction.GetFeatureValue("price").DoubleValue;
return(new MarsHabitatPricerOutput(priceValue));
}
public Task <ClassificationResult> Classify(Stream imageStream)
{
var imageData = NSData.FromStream(imageStream);
var uiImage = UIImage.LoadFromData(imageData);
var pixelBuffer = uiImage.Scale(new CGSize(224, 224)).ToCVPixelBuffer();
var result = _model.GetPrediction(new modelInput(pixelBuffer), out NSError error);
var classification = result.GetFeatureValue("classLabel").StringValue;
var loss = result.GetFeatureValue("loss").DictionaryValue;
var confidence = (NSNumber)loss.ValueForKey((NSString)classification);
return(Task.FromResult(new ClassificationResult(classification, confidence.DoubleValue)));
}
async Task FetchNonBatchResults(int num)
{
Stopwatch stopWatch = Stopwatch.StartNew();
await Task.Run(() =>
{
for (int i = 0; i < num; i++)
{
model.GetPrediction(inputs[i], out NSError error);
}
});
stopWatch.Stop();
nonBatchMilliseconds = stopWatch.ElapsedMilliseconds;
}
private IEnumerable <Recognition> Recognize(Stream source)
{
var byProbability = new List <Recognition>();
var image = UIImage.LoadFromData(NSData.FromStream(source));
if (model == null)
{
//ErrorOccurred(this, new EventArgsT<string>(error.ToString()));
return(byProbability);
}
var pixelBuffer = image.Scale(ImageSize).ToCVPixelBuffer();
var imageValue = MLFeatureValue.Create(pixelBuffer);
var inputs = new NSDictionary <NSString, NSObject>(new NSString(INPUT_NAME), imageValue);
var inputFeatureProvider = new MLDictionaryFeatureProvider(inputs, out var error1);
if (error1 != null)
{
//ErrorOccurred(this, new EventArgs<string>(error1.ToString()));
return(byProbability);
}
var outFeatures = model.GetPrediction(inputFeatureProvider, out var error2);
if (error2 != null)
{
//ErrorOccurred(this, new EventArgs<string>(error2.ToString()));
return(byProbability);
}
var predictionsDictionary = outFeatures.GetFeatureValue(OUTPUT_NAME).DictionaryValue;
foreach (var key in predictionsDictionary.Keys)
{
var description = (string)(NSString)key;
var prob = (double)predictionsDictionary[key];
byProbability.Add(new Recognition
{
Tag = description,
Probability = prob
});
}
//Sort descending
byProbability.Sort((t1, t2) => (t1.Probability.CompareTo(t2.Probability)) * -1);
return(byProbability);
}
/// <summary>
/// Make a prediction using the standard interface
/// </summary>
/// <param name="input">an instance of ObjectDetectorInput to predict from</param>
/// <param name="error">If an error occurs, upon return contains an NSError object that describes the problem.</param>
public ObjectDetectorOutput GetPrediction(ObjectDetectorInput input, out NSError error)
{
var prediction = model.GetPrediction(input, out error);
if (prediction == null)
{
return(null);
}
var confidenceValue = prediction.GetFeatureValue("confidence").MultiArrayValue;
var coordinatesValue = prediction.GetFeatureValue("coordinates").MultiArrayValue;
return(new ObjectDetectorOutput(confidenceValue, coordinatesValue));
}
/// <summary>
/// Make a prediction using the standard interface
/// </summary>
/// <param name="input">an instance of SqueezeNetInput to predict from</param>
/// <param name="error">If an error occurs, upon return contains an NSError object that describes the problem.</param>
public SqueezeNetOutput GetPrediction(SqueezeNetInput input, out NSError error)
{
var prediction = model.GetPrediction(input, out error);
if (prediction == null)
{
return(null);
}
var classLabelProbs = prediction.GetFeatureValue("classLabelProbs").DictionaryValue;
var classLabel = prediction.GetFeatureValue("classLabel").StringValue;
return(new SqueezeNetOutput(classLabelProbs, classLabel));
}
/// <summary>
/// Make a prediction using the standard interface
/// </summary>
/// <param name="input">an instance of ImageClassifierInput to predict from</param>
/// <param name="error">If an error occurs, upon return contains an NSError object that describes the problem.</param>
public ImageClassifierOutput GetPrediction(ImageClassifierInput input, out NSError error)
{
var prediction = model.GetPrediction(input, out error);
if (prediction == null)
{
return(null);
}
var classLabelProbsValue = prediction.GetFeatureValue("classLabelProbs").DictionaryValue;
var classLabelValue = prediction.GetFeatureValue("classLabel").StringValue;
return(new ImageClassifierOutput(classLabelProbsValue, classLabelValue));
}
/// <summary>
/// Extract the direct output from MLCore without post-processing
/// </summary>
/// <returns>A tuple contain the two matrices output by the model. First matrix: the box prediction encoding; second matrix: the class logit for each box.</returns>
/// <param name="input"> The formatted matrix representation of image after proper transpose and normalization</param>
private (MLMultiArray, MLMultiArray) RawPredict(MLMultiArray input)
{
var inputFeatureProvider = new ImageInputFeatureProvider(_inputFeatureName)
{
ImagePixelData = input
};
var resultBundle = _mlModel.GetPrediction(inputFeatureProvider, out _);
var output1 = resultBundle.GetFeatureValue(Constants.ModelOutputBoxFeatureName).MultiArrayValue;
var output2 = resultBundle.GetFeatureValue(Constants.ModelOutputClassFeatureName).MultiArrayValue;
inputFeatureProvider.Dispose();
return(output1, output2);
}
/// <summary>
/// Make a prediction using the standard interface
/// </summary>
/// <param name="input">an instance of mymodelInput to predict from</param>
/// <param name="error">If an error occurs, upon return contains an NSError object that describes the problem.</param>
public mymodelOutput GetPrediction(mymodelInput input, out NSError error)
{
var prediction = model.GetPrediction(input, out error);
if (prediction == null)
{
return(null);
}
var lossValue = prediction.GetFeatureValue("loss").DictionaryValue;
var classLabelValue = prediction.GetFeatureValue("classLabel").StringValue;
return(new mymodelOutput(lossValue, classLabelValue));
}
void updatePredictedPrice()
{
var pricerInput = new MarsHabitatPricerInput(datasource.GetValue(pickerView.SelectedRowInComponent(0), Feature.SolarPanels),
datasource.GetValue(pickerView.SelectedRowInComponent(1), Feature.Greenhouses),
datasource.GetValue(pickerView.SelectedRowInComponent(2), Feature.Size));
// Use the ML model
NSError prErr;
var outFeatures = model.GetPrediction(pricerInput, out prErr);
var result = outFeatures.GetFeatureValue("price").DoubleValue;
priceLabel.Text = "Predicted Price (millions) " + priceFormatter.StringFor(new NSNumber(result));
Console.WriteLine(prErr == null ? $"result was {result}" : "Unexpected runtime error " + prErr.Description);
}
public async Task <IEnumerable <Recognition> > RecognizeAsync(Stream source, params string[] parameters)
{
var results = new List <Recognition>();
var image = await UIImage.LoadFromData(NSData.FromStream(source)).ResizeImageAsync(INPUT_WIDTH, INPUT_HEIGHT);
await Task.Run(() =>
{
var pixelBuffer = image.Scale(ImageSize).ToCVPixelBuffer();
var imageValue = MLFeatureValue.Create(pixelBuffer);
var inputs = new NSDictionary <NSString, NSObject>(new NSString(INPUT_NAME), imageValue);
var inputFeatureProvider = new MLDictionaryFeatureProvider(inputs, out var error1);
if (error1 != null)
{
throw new ClassifierException(error1.ToString());
}
var outFeatures = model.GetPrediction(inputFeatureProvider, out var error2);
if (error2 != null)
{
throw new ClassifierException(error2.ToString());
}
var predictionsDictionary = outFeatures.GetFeatureValue(OUTPUT_NAME).DictionaryValue;
foreach (var key in predictionsDictionary.Keys)
{
var description = (string)(NSString)key;
var probability = (double)predictionsDictionary[key];
results.Add(new Recognition
{
Tag = description,
Probability = probability
});
}
// Sort descending.
results.Sort((t1, t2) => (t1.Probability.CompareTo(t2.Probability)) * -1);
});
return(results);
}
void MakePrediction(UIImage image)
{
//
// Get an input from the image
//
IMLFeatureProvider input = CreateInput(image);
//
// Predict what's in the image
//
var output = model.GetPrediction(input, out var error);
if (error != null)
{
Console.WriteLine($"Error predicting: {error}");
return;
}
var classLabel = output.GetFeatureValue("classLabel").StringValue;
//
// Display everything
//
Console.WriteLine($"Recognized:");
foreach (NSString n in output.FeatureNames)
{
var value = output.GetFeatureValue(n);
Console.WriteLine($" {n} == {value}");
}
var message = $"{classLabel.ToUpperInvariant ()} with probability {output.GetFeatureValue ("classLabelProbs").DictionaryValue[classLabel]}";
Console.WriteLine(message);
BeginInvokeOnMainThread(() => {
var alert = UIAlertController.Create(message, "I hope it's right!", UIAlertControllerStyle.Alert);
alert.AddAction(UIAlertAction.Create("OK", UIAlertActionStyle.Default, (obj) => {
DismissViewController(true, null);
}));
PresentViewController(alert, true, null);
});
}
internal void Classify(UIImage source)
{
var pixelBuffer = source.Scale(sizeForModel).ToCVPixelBuffer();
var imageValue = MLFeatureValue.Create(pixelBuffer);
var inputs = new NSDictionary <NSString, NSObject>(new NSString("image"), imageValue);
NSError error, error2;
var inputFp = new MLDictionaryFeatureProvider(inputs, out error);
if (error != null)
{
ErrorOccurred(this, new EventArgsT <string>(error.ToString()));
return;
}
var outFeatures = currentModel.GetPrediction(inputFp, out error2);
if (error2 != null)
{
ErrorOccurred(this, new EventArgsT <string>(error2.ToString()));
return;
}
var predictionsDictionary = outFeatures.GetFeatureValue("classLabelProbs").DictionaryValue;
var byProbability = new List <Tuple <double, string> >();
foreach (var key in predictionsDictionary.Keys)
{
var description = (string)(NSString)key;
var prob = (double)predictionsDictionary[key];
byProbability.Add(new Tuple <double, string>(prob, description));
}
//Sort descending
byProbability.Sort((t1, t2) => t1.Item1.CompareTo(t2.Item1) * -1);
var prediction = new ImageDescriptionPrediction();
prediction.ModelName = "VGG16";
prediction.predictions = byProbability;
PredictionsUpdated(this, new EventArgsT <ImageDescriptionPrediction>(prediction));
}
public ImageClassificationResult RecognizeImage(UIImage imageSource)
{
var pixelBuffer = imageSource.Scale(new CGSize(InputSize, InputSize)).ToCVPixelBuffer();
var imageValue = MLFeatureValue.Create(pixelBuffer);
var inputs = new NSDictionary <NSString, NSObject>(new NSString(InputName), imageValue);
NSError error, error2;
var inputFp = new MLDictionaryFeatureProvider(inputs, out error);
if (error != null)
{
Console.WriteLine("RecognizeImage Error 1: {0}", error);
return(null);
}
var outFeatures = _imageClassifierModel.GetPrediction(inputFp, out error2);
if (error2 != null)
{
Console.WriteLine("RecognizeImage Error 2: {0}", error2);
return(null);
}
var predictionsDictionary = outFeatures.GetFeatureValue(OutputName).DictionaryValue;
var outputs = new List <Tuple <double, string> >();
foreach (var key in predictionsDictionary.Keys)
{
var description = (string)(NSString)key;
var prob = (double)predictionsDictionary[key];
outputs.Add(new Tuple <double, string>(prob, description));
}
var results = outputs
.Select((t1, t2) => new ImageClassificationResult(t1.Item2, t1.Item1))
.ToList();
return(results.OrderByDescending(t => t.Probability).First());
}
public double UpdateEngagement(FaceAnchorReading f)
{
blink.Push((f.FacialExpressions["EyeBlinkLeft"].Value + f.FacialExpressions["EyeBlinkRight"].Value) / 2);
smile.Push((f.FacialExpressions["MouthSmileLeft"].Value + f.FacialExpressions["MouthSmileRight"].Value) / 2);
frown.Push((f.FacialExpressions["MouthFrownLeft"].Value + f.FacialExpressions["MouthFrownRight"].Value) / 2);
squint.Push((f.FacialExpressions["EyeSquintLeft"].Value + f.FacialExpressions["EyeSquintRight"].Value) / 2);
gazeIn.Push((f.FacialExpressions["EyeLookInLeft"].Value + f.FacialExpressions["EyeLookInRight"].Value) / 2);
gazeOut.Push((f.FacialExpressions["EyeLookOutLeft"].Value + f.FacialExpressions["EyeLookOutRight"].Value) / 2);
if (previousReading == null)
{
headSpeed.Push(0);
eyeDwell.Push(0);
headTilt.Push(0);
}
else
{
var deltaTime = f.ReadingTimestamp.Subtract(previousReading.ReadingTimestamp);
var deltaHead = VectorMagnitude(VectorDifference(previousReading.LookAtPointTransform, f.LookAtPointTransform));
headSpeed.Push((float)(deltaHead / deltaTime.TotalMilliseconds));
var leftEyeDisplacement = CoordinateDisplacement(f.LeftEyeTransform);
var rightEyeDisplacement = CoordinateDisplacement(f.RightEyeTransform);
var eyeDisplacement = Math.Abs(leftEyeDisplacement[0] + rightEyeDisplacement[0]) / 2;
eyeDwell.Push((float)(eyeDisplacement > 0 ? 1 / eyeDisplacement : 0));
headTilt.Push((float)(Math.Abs(leftEyeDisplacement[1] + rightEyeDisplacement[1]) / 2));
}
previousReading = f;
var modelInput = new CoreMLPathwayInput
{
PPI = PPI,
Blink = blink.Mean,
Smile = smile.Mean,
Frown = frown.Mean,
Squint = squint.Mean,
GazeIn = gazeIn.Mean,
GazeOut = gazeOut.Mean,
HeadSpeed = headSpeed.Mean,
EyeDwell = eyeDwell.Mean,
HeadTilt = headTilt.Mean
};
IMLFeatureProvider predictionOut = model.GetPrediction(modelInput, out _);
var targetFeatureValue = predictionOut.GetFeatureValue("target");
var prediction = targetFeatureValue.Int64Value;
var classProbabilityFeatureValue = predictionOut.GetFeatureValue("classProbability");
var probabilityx = classProbabilityFeatureValue.DictionaryValue.Values[0];
var probabilityy = classProbabilityFeatureValue.DictionaryValue.Values[1];
//// this is a bit hacky, but it's all relative
//value = (probabilityy.FloatValue - probabilityx.FloatValue - 0.5f) * 2f;
//value = (value < 0f) ? 0f : value;
value = probabilityy.FloatValue - probabilityx.FloatValue;
if (value < 0f)
{
Console.WriteLine(value + " negative !");
value = 0f;
}
else
{
Console.WriteLine(value);
}
return(value);
//return probabilityy.FloatValue;
//return prediction;
}
