public string getPrediction(Plugin.Media.Abstractions.MediaFile image)
{
Task <Bitmap> temp = getBitmap(image);
Bitmap bitmap = temp.Result;
var assets = Android.App.Application.Context.Assets;
TensorFlowInferenceInterface inferenceInterface = new TensorFlowInferenceInterface(assets, "frozen_inference_graph.pb");
var sr = new StreamReader(assets.Open("labels.txt"));
var labels = sr.ReadToEnd()
.Split('\n')
.Select(s => s.Trim())
.Where(s => !string.IsNullOrEmpty(s))
.ToList();
var outputNames = new[] { "detection_classes" };
var floatValues = GetBitmapPixels(bitmap);
var outputs = new float[labels.Count];
inferenceInterface.Feed("ToFloat", floatValues, 1, 227, 227, 3);
inferenceInterface.Run(outputNames);
inferenceInterface.Fetch("detection_classes", outputs);
var results = new List <Tuple <float, string> >();
for (var i = 0; i < outputs.Length; ++i)
{
results.Add(Tuple.Create(outputs[i], labels[i]));
}
return(results.OrderByDescending(t => t.Item1).First().Item2);
}
public Task LoadModelFromUrl(string url)
{
if (string.IsNullOrWhiteSpace(url))
{
throw new ArgumentNullException(nameof(url));
}
return(Task.Run(async() =>
{
// HACK: unzip model in azure indead of device and get the stream
url = $"{Constants.AzureFunctionsUrl}/MLExport?platform=tensorflow&format=binary&code={Constants.AzureFunctionsCode}";
// TODO: we still need labels
var client = new HttpClient();
var rawModelBytes = await client.GetByteArrayAsync("https://mobcatnotmobcat.azurewebsites.net/api/MLExport?platform=tensorflow&format=binary&code=rLzmW4glo3/KUahNnEHlI/fngVrlzQJ/9y4YIIHS6yHAtNKNHiCm0w==");
Stream zipModelStream = new MemoryStream(rawModelBytes);
inferenceInterface = new TensorFlowInferenceInterface(zipModelStream);
var assets = Android.App.Application.Context.Assets;
using (var sr = new StreamReader(assets.Open("labels2.txt")))
{
var content = sr.ReadToEnd();
_loadedLabels = content.Split('\n').Select(s => s.Trim()).Where(s => !string.IsNullOrEmpty(s)).ToList();
}
}));
}
internal void Init(string modelName, string labelsFileName, ModelType modelType)
{
_modelType = modelType;
try
{
var assets = Android.App.Application.Context.Assets;
using (var sr = new StreamReader(File.Exists(labelsFileName)? File.OpenRead(labelsFileName) : assets.Open(labelsFileName)))
{
var content = sr.ReadToEnd();
_labels = content.Split('\n').Select(s => s.Trim()).Where(s => !string.IsNullOrEmpty(s)).ToList();
}
_inferenceInterface = new TensorFlowInferenceInterface(assets, modelName);
InputSize = Convert.ToInt32(_inferenceInterface.GraphOperation(InputName).Output(0).Shape().Size(1));
var iter = _inferenceInterface.Graph().Operations();
while (iter.HasNext && !_hasNormalizationLayer)
{
var op = iter.Next() as Operation;
if (op.Name().Contains(DataNormLayerPrefix))
{
_hasNormalizationLayer = true;
}
}
}
catch (Exception ex)
{
throw new ImageClassifierException("Failed to load the model - check the inner exception for more details", ex);
}
}
public PhotoDetector()
{
_inferenceInterface = new TensorFlowInferenceInterface(CrossCurrentActivity.Current.Activity.Assets, ModelFile);
using (var sr = new StreamReader(CrossCurrentActivity.Current.Activity.Assets.Open(LabelFile)))
{
_labels = sr.ReadToEnd().Split('\n').Select(x => x.Trim()).Where(x => !string.IsNullOrEmpty(x)).ToArray();
}
}
public InceptionClassifier()
{
var assetManager = CrossCurrentActivity.Current.Activity.Assets;
labels = new List <string>();
labels.AddRange(ReadLabelsIntoMemory(assetManager, LABEL_FILE));
inferenceInterface = new TensorFlowInferenceInterface(assetManager, MODEL_FILE);
}
public TensorflowContribAndroidPipeline(string modelName, int modelInputSize, string inputName, string outputName)
{
var assets = Application.Context.Assets;
inferenceInterface = new TensorFlowInferenceInterface(assets, modelName);
ModelInputSize = modelInputSize;
InputName = inputName;
OutputName = outputName;
}
public void LoadModel(string path)
{
var name = Path.GetFileNameWithoutExtension(path);
path = name + ".pb";
if (path.Equals(LastModelLoaded))
{
return;
}
var info = CorePackage.Entity.Type.Resource.Instance.Directory + name + ".txt";
List <string> lines = new List <string>();
using (StreamReader sr = new StreamReader(info))
{
while (sr.Peek() >= 0)
{
lines.Add(sr.ReadLine());
}
}
_inputName = lines[0];
_outputName = lines[1];
var shape = lines[2].Replace(" ", "").Replace("(", "").Replace(")", "").Split(",");
_inputSize = int.Parse(shape[1]);
_lastDim = int.Parse(shape[3]);
CorePackage.Entity.Type.Resource.Instance.Directory = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData) + "/";
var modelFile = File.Open(CorePackage.Entity.Type.Resource.Instance.Directory + path, FileMode.Open);
try
{
//var assets = Android.App.Application.Context.Assets;
//using (var sr = new StreamReader(assets.Open(labelsFileName)))
//{
// var content = sr.ReadToEnd();
// _labels = content.Split('\n').Select(s => s.Trim()).Where(s => !string.IsNullOrEmpty(s)).ToList();
//}
_inferenceInterface = new TensorFlowInferenceInterface(modelFile);
LastModelLoaded = path;
var iter = _inferenceInterface.Graph().Operations();
while (iter.HasNext && !_hasNormalizationLayer)
{
var op = iter.Next() as Operation;
if (op.Name().Contains(DataNormLayerPrefix))
{
_hasNormalizationLayer = true;
}
}
}
catch (Exception ex)
{
throw new RuntimeException("Failed to load the model - check the inner exception for more details" + ex.Message);
}
}
public ImageClassifier()
{
var assets = Application.Context.Assets;
inferenceInterface = new TensorFlowInferenceInterface(assets, "model.pb");
using (var sr = new StreamReader(assets.Open("labels.txt")))
{
var content = sr.ReadToEnd();
labels = content.Split('\n').Select(s => s.Trim()).Where(s => !string.IsNullOrEmpty(s)).ToList();
}
}
/// <summary>
/// Initializes a new instance of the <see cref="TensorFlowImageModel"/> class and loads the TensorFlow model.
/// </summary>
/// <param name="assets">A reference to an <see cref="AssetManager"/> that will be used to load the model file.</param>
/// <param name="modelAssetPath">The path to the model file (e.g. "file:///android_asset/model.pb")</param>
/// <param name="inputTensorName">The name of the input Tensor of the model.</param>
/// <param name="outputNodeNames">The names of the output nodes of the model.</param>
/// <param name="outputSizes">The number of values expected in the array at each of the output nodes.</param>
public TensorFlowImageModel(AssetManager assets, string modelAssetPath, string inputTensorName, string[] outputNodeNames, int[] outputSizes)
{
if (outputNodeNames.Length != outputSizes.Length)
{
throw new ArgumentException("outputNodeNames (length=" + outputNodeNames.Length + ") and outputSizes (length=" + outputSizes.Length + ") must be of equal length");
}
this.tfInterface = new TensorFlowInferenceInterface(assets, modelAssetPath);
this.inputTensorName = inputTensorName;
this.outputNodeNames = outputNodeNames;
this.outputSizes = outputSizes;
}
public ImageClassifier()
{
var assets = Application.Context.Assets;
_inferenceInterface = new TensorFlowInferenceInterface(assets, ModelFileName);
using (var sr = new StreamReader(assets.Open(LabelsFileName)))
{
var content = sr.ReadToEnd();
_labels = LoadLabels(content);
}
}
public InceptionClassifier(AssetManager assetManager)
{
labels.AddRange(ReadLabelsIntoMemory(assetManager, LABEL_FILE));
inferenceInterface = new TensorFlowInferenceInterface(assetManager, MODEL_FILE);
var operation = inferenceInterface.GraphOperation(OUTPUT_NAME);
var numClasses = operation.Output(0).Shape().Size(1);
//TODO: Confirm in Android, because my results are 1001 labels, 1008 classes
System.Console.WriteLine($"Read {labels.Count} labels, output layer size is {numClasses}");
var bmp = BitmapCreate(assetManager, "husky.png");
var results = Recognize(bmp);
}
public static TensorflowModel CreateTensorflowModel()
{
var assets = Application.Context.Assets;
var inferenceInterface = new TensorFlowInferenceInterface(assets, ModelPath);
IEnumerable <string> labels;
using (var sr = new StreamReader(assets.Open(LabelsPath)))
{
var content = sr.ReadToEnd();
labels = content.Split('\n').Select(s => s.Trim()).Where(s => !string.IsNullOrEmpty(s)).ToList();
}
return(new TensorflowModel(labels.ToArray(), inferenceInterface));
}
public void Classify(byte[] bytes)
{
var assets = Application.Context.Assets;
var inferenceInterface = new TensorFlowInferenceInterface(assets, "people-or-not-model.pb");
var streamReader = new StreamReader(assets.Open("people-or-not-labels.txt"));
var labels = streamReader
.ReadToEnd()
.Split('\n')
.Select(s => s.Trim())
.Where(s => !string.IsNullOrEmpty(s))
.ToList();
//page 354
var bitmap = BitmapFactory.DecodeByteArray(bytes, 0, bytes.Length);
var resizedBitMap = Bitmap.CreateScaledBitmap(bitmap, 227, 227, false).Copy(Bitmap.Config.Argb8888, false);
var floatValues = new float[227 * 227 * 3];
var intValues = new int[227 * 227 * 3];
resizedBitMap.GetPixels(intValues, 0, 227, 0, 0, 227, 227);
for (int i = 0; i < intValues.Length; i++)
{
var val = intValues[i];
floatValues[i * 3 + 0] = ((val & 0xFF) - 104);
floatValues[i * 3 + 1] = ((val & 8) - 117);
floatValues[i * 3 + 2] = ((val & 16) - 123);
}
var outputs = new float[labels.Count];
inferenceInterface.Feed("Placeholder", floatValues, 1, 227, 227, 3);
inferenceInterface.Run(new[] { "loss" });
inferenceInterface.Fetch("loss", outputs);
var results = new Dictionary <string, float>();
for (var i = 0; i < labels.Count; i++)
{
var label = labels[i];
results.Add(label, outputs[i]);
}
ClassificationCompleted?.Invoke(this, new ClassificationEventArgs(results));
}
public async Task InitializeAsync(ModelType modelType, params string[] parameters)
{
var assets = Android.App.Application.Context.Assets;
var modelFile = $"file:///android_asset/{parameters[0]}";
var labelFile = parameters[1];
labels = new List <string>();
using var sr = new StreamReader(assets.Open(labelFile));
var content = await sr.ReadToEndAsync();
var labelStrings = content.Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries).Select(e => e.TrimEnd('\r'));
labels.AddRange(labelStrings);
inferenceInterface = new TensorFlowInferenceInterface(assets, modelFile);
inputSize = (int)inferenceInterface.GraphOperation(INPUT_NAME).Output(0).Shape().Size(1);
}
public Task LoadModel()
{
if (_loadedLabels != null)
{
return(Task.FromResult(true));
}
return(Task.Run(() =>
{
//loading model and labels
var assets = Android.App.Application.Context.Assets;
inferenceInterface = new TensorFlowInferenceInterface(assets, "model.pb");
using (var sr = new StreamReader(assets.Open("labels.txt")))
{
var content = sr.ReadToEnd();
_loadedLabels = content.Split('\n').Select(s => s.Trim()).Where(s => !string.IsNullOrEmpty(s)).ToList();
}
}));
}
public override void Init(string modelName, ModelType modelType, int inputSize = 227, string inputName = "Placeholder", string outputName = "loss")
{
base.Init(modelName, modelType, inputSize, inputName, outputName);
try
{
var assets = Android.App.Application.Context.Assets;
using (var sr = new StreamReader(assets.Open("labels.txt")))
{
var content = sr.ReadToEnd();
_labels = content.Split('\n').Select(s => s.Trim()).Where(s => !string.IsNullOrEmpty(s)).ToList();
}
_inferenceInterface = new TensorFlowInferenceInterface(assets, modelName);
}
catch (Exception ex)
{
throw new ImageClassifierException("Failed to load the model - check the inner exception for more details", ex);
}
}
public async Task InitializeAsync(ModelType modelType, params string[] parameters)
{
ImageMeanR = modelType.ImageMeanR();
ImageMeanG = modelType.ImageMeanG();
ImageMeanB = modelType.ImageMeanB();
var assets = Android.App.Application.Context.Assets;
var modelFile = $"file:///android_asset/{parameters[0]}";
var labelFile = parameters[1];
labels = new List <string>();
using (var sr = new StreamReader(assets.Open(labelFile)))
{
var content = await sr.ReadToEndAsync();
var labelStrings = content.Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries).Select(e => e.TrimEnd('\r'));
labels.AddRange(labelStrings);
}
inferenceInterface = new TensorFlowInferenceInterface(assets, modelFile);
}
protected override void OnActivityResult(int requestCode, [GeneratedEnum] Result resultCode, Intent data)
{
base.OnActivityResult(requestCode, resultCode, data);
Bitmap bitmap = (Bitmap)data.Extras.Get("data");
imageView.SetImageBitmap(bitmap);
var assets = Application.Context.Assets;
var inferenceInterface = new TensorFlowInferenceInterface(assets, "model.pb");
var sr = new StreamReader(assets.Open("labels.txt"));
var labels = sr.ReadToEnd()
.Split('\n')
.Select(s => s.Trim())
.Where(s => !string.IsNullOrEmpty(s))
.ToList();
var floatValues = GetBitmapPixels(bitmap);
var outputs = new float[labels.Count];
inferenceInterface.Feed("Placeholder", floatValues, 1, 224, 224, 3);
inferenceInterface.Run(new[] { "loss" });
inferenceInterface.Fetch("loss", outputs);
var results = new Recognition[labels.Count];
for (int i = 0; i < labels.Count; i++)
{
results[i] = new Recognition {
Confidence = outputs[i], Label = labels[i]
};
}
Array.Sort(results, (x, y) => y.Confidence.CompareTo(x.Confidence));
var result = String.Format("I think the cat on this picture is: {0}. I'm {1} confident", results[0].Label, results[0].Confidence.ToString("P1"));
((TextView)FindViewById(Resource.Id.result)).Text = result;
//Task.Run(() => TextToSpeechAsync(result));
//TextToSpeech.SpeakAsync(result).Wait(5000);
TextToSpeech.SpeakAsync(result).ContinueWith((b) => { Task.Run(() => TextToSpeechAsync(result)); });
}
public async void Classification(byte[] bytes)
{
var assets = Application.Context.Assets;
var inferenceInterface = new TensorFlowInferenceInterface(assets, "model.pb");
var inputSize = (int)inferenceInterface.GraphOperation("Placeholder").Output(0).Shape().Size(1);
List <string> labels;
using (var streamReader = new StreamReader(assets.Open("labels.txt")))
{
labels = streamReader.ReadToEnd().Split('\n').Select(s => s.Trim()).Where(s => !string.IsNullOrEmpty(s))
.ToList();
}
var bitmap = await BitmapFactory.DecodeByteArrayAsync(bytes, 0, bytes.Length);
var resizedMap = Bitmap.CreateScaledBitmap(bitmap, inputSize, inputSize, false)
.Copy(Bitmap.Config.Argb8888, false);
var floatValues = new float[inputSize * inputSize * 3];
var intValues = new int[inputSize * inputSize];
resizedMap.GetPixels(intValues, 0, inputSize, 0, 0, inputSize, inputSize);
for (var i = 0; i < intValues.Length; ++i)
{
var intValue = intValues[i];
floatValues[i * 3 + 0] = (intValue & 0xFF) - 105f;
floatValues[i * 3 + 1] = ((intValue >> 8) & 0xFF) - 117f;
floatValues[i * 3 + 2] = ((intValue >> 16) & 0xFF) - 124f;
}
var operation = inferenceInterface.GraphOperation("loss");
var outputs = new float[labels.Count];
inferenceInterface.Feed("Placeholder", floatValues, 1, inputSize, inputSize, 3);
inferenceInterface.Run(new[] { "loss" }, true);
inferenceInterface.Fetch("loss", outputs);
var result = new Dictionary <string, float>();
for (var i = 0; i < labels.Count; i++)
{
var label = labels[i];
result.Add(label, outputs[i]);
}
var maxConf = 0f;
for (var i = 0; i < outputs.Length; ++i)
{
if (outputs[i] > maxConf)
{
maxConf = outputs[i];
}
}
ClassificationCompleted?.Invoke(this, new ClassificationEventArgs(result));
}
private TensorflowModel(string[] labels, TensorFlowInferenceInterface inferenceInterface)
{
this.labels = labels;
this.inferenceInterface = inferenceInterface;
}
protected override void OnCreate(Bundle savedInstanceState)
{
base.OnCreate(savedInstanceState);
// Set our view from the "main" layout resource
SetContentView(Resource.Layout.Main);
// Get our button from the layout resource,
// and attach an event to it
Button button = FindViewById <Button>(Resource.Id.myButton);
button.Click += delegate { button.Text = $"{count++} clicks!"; };
try
{
TensorFlowInferenceInterface tfi = new TensorFlowInferenceInterface(Assets, "file:///android_asset/TF_LSTM_Inference.pb");
float[] inputSeaLevels = new float[] {
4.92F, 2.022F, -0.206F, 2.355F, 4.08F, 1.828F, -0.005F, 2.83F,
4.966F, 2.715F, -0.073F, 1.69F, 3.958F, 2.5F, 0.201F, 2.075F, 4.754F, 3.475F,
0.345F, 0.954F, 3.665F, 3.165F, 0.562F, 1.285F, 4.415F, 4.083F, 0.83F, 0.327F,
3.304F, 3.589F, 0.976F, 0.707F, 3.989F, 4.37F, 1.375F, 0.039F, 2.863F, 3.715F,
1.525F, 0.507F, 3.403F, 4.38F, 2.06F, 0.097F, 2.251F, 3.671F, 2.223F, 0.579F,
2.605F, 4.25F, 2.791F, 0.306F, 1.489F, 3.575F, 2.907F, 0.759F, 1.741F, 4.018F,
3.381F, 0.589F, 0.796F, 3.382F, 3.441F, 1.103F, 1.099F, 3.566F, 3.75F, 1.09F,
0.393F, 2.906F, 3.831F, 1.77F, 0.816F, 2.739F, 3.922F, 1.891F, 0.285F, 2.02F,
4.083F, 2.727F, 0.833F, 1.593F, 3.863F, 2.824F, 0.408F, 0.906F, 4.071F, 3.738F,
1.149F, 0.511F, 3.431F, 3.626F, 0.863F, -0.004F, 3.57F, 4.568F, 1.906F,
-0.079F, 2.489F, 4.115F, 1.777F, -0.396F, 2.456F, 5.046F, 3.106F, -0.04F,
1.113F, 4.168F, 2.998F, -0.246F, 0.941F, 5.009F, 4.425F, 0.516F, -0.267F,
3.695F, 4.092F, 0.405F, -0.384F, 4.32F, 5.429F, 1.541F, -1.036F, 2.628F,
4.711F, 1.479F, -0.977F, 3.043F, 5.82F, 2.856F, -0.932F, 1.249F, 4.713F,
2.828F, -0.734F, 1.438F, 5.546F, 4.182F, -0.159F, -0.109F, 4.137F, 4.03F,
0.09F, 0.042F, 4.693F, 5.075F, 0.921F, -0.925F, 3.14F, 4.67F, 1.227F, -0.599F,
3.458F, 5.274F, 2.089F, -0.893F, 1.945F, 4.618F, 2.489F, -0.336F, 2.106F,
4.828F, 3.178F, -0.192F, 0.816F, 4.026F, 3.61F, 0.491F, 0.961F, 3.972F, 3.93F,
0.717F, 0.057F, 3.155F, 4.273F, 1.457F, 0.369F, 2.96F, 4.147F, 1.559F, -0.088F,
2.226F, 4.349F, 2.398F, 0.485F, 1.964F, 3.861F, 2.329F, 0.335F, 1.357F, 3.972F,
3.286F, 1.092F, 1.083F, 3.277F, 3.011F, 0.981F, 0.632F, 3.375F, 3.987F
};
String INPUT_ARGUMENT_NAME = "lstm_1_input";
String OUTPUT_VARIABLE_NAME = "output_node0";
int OUTPUT_SIZE = 100;
tfi.Feed(INPUT_ARGUMENT_NAME, inputSeaLevels, inputSeaLevels.Length, 1, 1);
tfi.Run(new String[] { OUTPUT_VARIABLE_NAME });
float[] predictions = new float[OUTPUT_SIZE];
tfi.Fetch(OUTPUT_VARIABLE_NAME, predictions);
button.Text = predictions[0].ToString();
}
catch (Exception x)
{
Console.WriteLine(x);
}
// Seriously, it's this easy?
using (var ic = new InceptionClassifier(BaseContext.Assets))
{
var results = ic.Recognize(BitmapCreate(BaseContext.Assets, "husky.png"));
var top = results.First();
// TextView label = FindViewById<TextView>(Resource.Id.)
}
}
public TensorflowInferencePredictor(AssetManager assetManager)
{
inferenceInterface = new TensorFlowInferenceInterface(assetManager, MODEL_FILE_URL);
}
