public IActionResult ClassifyImage()
{
string modelFile = Path.Combine(AppContext.BaseDirectory, modelFolder, enginesConfig.ModelName);
string imagePath = Path.Combine(AppContext.BaseDirectory, "TestImages", "TestImage.png");
Image image = Image.FromFile(imagePath);
using var ms = new MemoryStream();
image.Save(ms, image.RawFormat);
var imageByte = ms.ToArray();
InMemoryImageData imageData = new InMemoryImageData(imageByte, "image.jpg", string.Empty);
if (!System.IO.File.Exists(modelFile))
{
return(NotFound());
}
Console.WriteLine($"Memory before loading engine 0: {Process.GetCurrentProcess().WorkingSet64 / (double)(1024 * 1024 * 1024):0.####} GB");
// create instances of engine and load them into the memory
List <PredictionEngine <InMemoryImageData, ImagePred> > engines = LoadPool(modelFile);
// every first call of Predict() of the prediction engine instance increases the memory by approximately 500MB. This is too much!!!
CallPredicts(engines, imageData);
// subsequent call of Predict() of engine behaves normally
CallPredicts(engines, imageData);
return(Ok($"The test is finished. :). Total memory consumption: {Process.GetCurrentProcess().WorkingSet64 / (double)(1024 * 1024 * 1024):0.####} GB"));
}
private static void PrintInformationAboutPrediction(InMemoryImageData imageToPredict, ImagePrediction prediction)
{
Console.WriteLine($"Image Filename : [{imageToPredict.ImageFileName}], " +
$"Predicted Label : [{prediction.PredictedLabel}], " +
$"Probability : [{prediction.Score.Max()}] "
);
}
public ImagePrediction Predict(InMemoryImageData data)
{
DateTime started = DateTime.Now;
var result = this.PredictionEngine.Predict(data);
result.Seconds = (int)(started - DateTime.Now).TotalSeconds;
return(result);
}
public async Task <IActionResult> ClassifyImage([FromForm] IFormFile Image)
{
if (Image.Length == 0)
{
return(BadRequest());
}
MemoryStream imageMemoryStream = new MemoryStream();
await Image.CopyToAsync(imageMemoryStream);
byte[] imageData = imageMemoryStream.ToArray();
if (!imageData.IsValidImage())
{
return(StatusCode(StatusCodes.Status415UnsupportedMediaType));
}
_logger.LogInformation("Start Processing Image ....");
Stopwatch watch = Stopwatch.StartNew();
InMemoryImageData imageInputData = new InMemoryImageData(image: imageData, label: null, imageFileName: null);
var prediction = _predictionEnginePool.Predict(imageInputData);
watch.Stop();
var elapsedMs = watch.ElapsedMilliseconds;
_logger.LogInformation($"Image processed in { elapsedMs } miliseconds");
string filePath = Path.Combine(_environment.ContentRootPath, "wwwroot", "Images", Image.FileName);
using (var fileStream = new FileStream(filePath, FileMode.Create, FileAccess.Write))
{
await Image.CopyToAsync(fileStream);
}
ImagePredictedLabelWithProbability imageBestLabelPrediction = new ImagePredictedLabelWithProbability
{
PredictedLabel = prediction.PredictedLabel,
Probability = prediction.Score.Max(),
PredictionExecutionTime = elapsedMs,
ImageId = Image.FileName
};
return(Ok(imageBestLabelPrediction));
}
public async Task <IActionResult> ClassifyImage(IFormFile imageFile)
{
if (imageFile.Length == 0)
{
return(BadRequest());
}
var imageMemoryStream = new MemoryStream();
await imageFile.CopyToAsync(imageMemoryStream);
// Check that the image is valid.
byte[] imageData = imageMemoryStream.ToArray();
if (!imageData.IsValidImage())
{
return(StatusCode(StatusCodes.Status415UnsupportedMediaType));
}
_logger.LogInformation("Start processing image...");
// Measure execution time.
var watch = System.Diagnostics.Stopwatch.StartNew();
// Set the specific image data into the ImageInputData type used in the DataView.
var imageInputData = new InMemoryImageData {
Image = imageData
};
// Predict code for provided image.
var prediction = _predictionEnginePool.Predict(imageInputData);
// Stop measuring time.
watch.Stop();
var elapsedMs = watch.ElapsedMilliseconds;
_logger.LogInformation($"Image processed in {elapsedMs} miliseconds");
// Predict the image's label (The one with highest probability).
ImagePredictedLabelWithProbability imageBestLabelPrediction =
new ImagePredictedLabelWithProbability()
{
PredictedLabel = prediction.PredictedLabel,
Probability = prediction.Score.Max(),
PredictionExecutionTime = elapsedMs,
ImageId = imageFile.FileName
};
return(Ok(imageBestLabelPrediction));
}
public string WhatIsOnImage(byte[] rawData, string imageName)
{
var mlContext = new MLContext();
mlContext.Log += (object sender, LoggingEventArgs e) => { };
var trainedModel = mlContext.Model.Load(_modelPath, out _);
var predictionEngine = mlContext.Model.CreatePredictionEngine <InMemoryImageData, ImagePrediction>(trainedModel);
var testImage = new InMemoryImageData(rawData, "", imageName);
var prediction = predictionEngine.Predict(testImage);
return(prediction.PredictedLabel);
}
public static void WarmUpPredictionEnginePool(this IServiceCollection services, string warmupImage, byte additionalTestCount = 5)
{
var predictionEnginePool =
services.BuildServiceProvider()
.GetRequiredService <PredictionEnginePool <InMemoryImageData, ImagePrediction> >();
var predictionEngine = predictionEnginePool.GetPredictionEngine();
predictionEnginePool.ReturnPredictionEngine(predictionEngine);
// Get a sample image
var img = Image.FromFile(warmupImage);
byte[] imageData;
using (MemoryStream ms = new MemoryStream())
{
img.Save(ms, System.Drawing.Imaging.ImageFormat.Png);
imageData = ms.ToArray();
}
var imageInputData = new InMemoryImageData(image: imageData, label: null, imageFileName: null);
// Measure execution time.
var watch = System.Diagnostics.Stopwatch.StartNew();
ImagePrediction prediction = predictionEnginePool.Predict(imageInputData);
for (int i = 1; i < additionalTestCount; i++)
{
predictionEnginePool.Predict(imageInputData);
}
// Stop measuring time.
watch.Stop();
var elapsedMs = watch.ElapsedMilliseconds;
Console.WriteLine();
Console.WriteLine(new string('*', 25));
Console.WriteLine();
Console.WriteLine("Warmup prediction: {0}, {1}% - {2} sec.",
prediction.PredictedLabel,
prediction.Score.Max() * 100,
elapsedMs / 1000.0);
Console.WriteLine();
Console.WriteLine(new string('*', 25));
Console.WriteLine();
}
public async Task <ClassificationPredictionVM> Classify(string classifier, IFormFile imageFile)
{
if (imageFile.Length == 0)
{
throw new EmptyFileException();
}
var imageMemoryStream = new MemoryStream();
await imageFile.CopyToAsync(imageMemoryStream);
var imageData = imageMemoryStream.ToArray();
if (!imageData.IsValidImage())
{
throw new ImageFormatException(ImageExtensions.CanBeUsed.Select(x => x.GetDescription()));
}
_logger.LogInformation("Start processing image...");
var watch = System.Diagnostics.Stopwatch.StartNew();
var imageInputData = new InMemoryImageData(imageData);
_predictionEnginePoolService.Get(classifier, out PredictionEngine <InMemoryImageData, ImagePrediction> engine);
if (engine is null)
{
throw new NotFoundClassifierException($"Classifier `{classifier}` is not found");
}
var prediction = engine.Predict(imageInputData);
watch.Stop();
var elapsedMs = watch.ElapsedMilliseconds;
_logger.LogInformation($"Image processed in {elapsedMs} miliseconds");
var imageBestLabelPrediction = new ClassificationPredictionVM
{
PredictedLabel = prediction.PredictedLabel,
Probability = prediction.Score.Max(),
PredictionExecutionTime = elapsedMs,
ImageId = imageFile.FileName,
};
return(imageBestLabelPrediction);
}
private static void TrySinglePrediction(string imagesForPredictions,
MLContext mlContext, ITransformer trainedModel)
{
// Create prediction function to try one prediction
var predictionEngine = mlContext.Model
.CreatePredictionEngine <InMemoryImageData, ImagePrediction>(trainedModel);
IEnumerable <InMemoryImageData> testImages = LoadInMemoryImagesFromDirectory(
imagesForPredictions, false);
InMemoryImageData imageToPredict = new InMemoryImageData
{
Image = testImages.First().Image
};
var prediction = predictionEngine.Predict(imageToPredict);
Console.WriteLine($"Scores : [{string.Join(",", prediction.Score)}], " +
$"Predicted Label : {prediction.PredictedLabel}");
}
public ImagePrediction Predict(string location)
{
FileInfo file = new FileInfo(location);
if (!file.Exists)
{
throw new Exception("No file");
}
if (!file.Extension.Equals(".jpg") && !file.Extension.Equals(".png"))
{
throw new Exception("Only jpg and png");
}
var data = new InMemoryImageData(
image: File.ReadAllBytes(location),
label: "",
imageFileName: Path.GetFileName(location));
return(this.Predict(data));
}
private void CallPredicts(List <PredictionEngine <InMemoryImageData, ImagePred> > predictionEngines, InMemoryImageData imageData)
{
int i = 0;
foreach (var engine in predictionEngines)
{
var result = engine.Predict(imageData);
Console.WriteLine($"Memory after calling Predict of engine {i++}: {Process.GetCurrentProcess().WorkingSet64 / (double)(1000 * 1000 * 1000):0.####} GB");
}
}
/// <summary>
/// Do second prediction for count the taken time.
/// </summary>
/// <param name="predictionEngine"></param>
/// <param name="imageToPredict"></param>
private static void DoSecondPrediction(PredictionEngine <InMemoryImageData, ImagePrediction> predictionEngine, InMemoryImageData imageToPredict)
{
var watch2 = System.Diagnostics.Stopwatch.StartNew();
predictionEngine.Predict(imageToPredict);
watch2.Stop();
var elapsedMs2 = watch2.ElapsedMilliseconds;
Console.WriteLine("Second Prediction took: " + elapsedMs2 + "mlSecs");
}
/// <summary>
/// Get the prediction and count the taken time.
/// </summary>
/// <param name="predictionEngine"></param>
/// <param name="imageToPredict"></param>
/// <returns>Prediction</returns>
private static ImagePrediction GetFirstPrediction(PredictionEngine <InMemoryImageData, ImagePrediction> predictionEngine, InMemoryImageData imageToPredict)
{
var watch = System.Diagnostics.Stopwatch.StartNew();
var prediction = predictionEngine.Predict(imageToPredict);
watch.Stop();
var elapsedMs = watch.ElapsedMilliseconds;
Console.WriteLine("First Prediction took: " + elapsedMs + "mlSecs");
return(prediction);
}
static void Main(string[] args)
{
//03 準備環境與 load 檔案
string dataPath = Path.Combine(Environment.CurrentDirectory, @"../../../data");
string imageFolderPath = Path.Combine(dataPath, "mask_dataset");
var mlContext = new MLContext(seed: 1); //
IEnumerable <ImageData> images = LoadImagesFromDirectory( // 讀出檔案路徑清單
folder: imageFolderPath,
useFolderNameAsLabel: true);
IDataView fullImagesDataset = mlContext.Data.LoadFromEnumerable(images); //載入影像
IDataView shuffledFullImageFilePathsDataset = mlContext.Data.ShuffleRows(fullImagesDataset);
//04 數據前處理
IDataView shuffledFullImagesDataset = mlContext.Transforms.Conversion.
MapValueToKey(outputColumnName: "LabelAsKey", inputColumnName: "Label", keyOrdinality: KeyOrdinality.ByValue) // label 轉成 index key
.Append(mlContext.Transforms.LoadRawImageBytes(
outputColumnName: "Image",
imageFolder: imageFolderPath,
inputColumnName: "ImagePath"))
.Fit(shuffledFullImageFilePathsDataset)
.Transform(shuffledFullImageFilePathsDataset);
//05 切割資料
var trainTestData = mlContext.Data.TrainTestSplit(shuffledFullImagesDataset, testFraction: 0.2);
IDataView trainDataView = trainTestData.TrainSet;
IDataView testDataView = trainTestData.TestSet;
//06 Train model
var options = new ImageClassificationTrainer.Options()
{
FeatureColumnName = "Image",
LabelColumnName = "LabelAsKey",
Arch = ImageClassificationTrainer.Architecture.MobilenetV2, //選用的模型, 會直接上網下載
Epoch = 50,
BatchSize = 10,
LearningRate = 0.01f,
MetricsCallback = (metrics) => Console.WriteLine(metrics),
ValidationSet = testDataView
};
var pipeline = mlContext.MulticlassClassification.Trainers.ImageClassification(options)
.Append(mlContext.Transforms.Conversion.MapKeyToValue(
outputColumnName: "PredictedLabel",
inputColumnName: "PredictedLabel"));
ITransformer trainedModel = pipeline.Fit(trainDataView);
var predictionsDataView = trainedModel.Transform(testDataView);
//07 Evaluate
var metrics = mlContext.MulticlassClassification.Evaluate(predictionsDataView,
labelColumnName: "LabelAsKey", predictedLabelColumnName: "PredictedLabel");
Console.WriteLine($"************************************************************");
Console.WriteLine($"* Metrics for TensorFlow DNN Transfer Learning multi-class classification model ");
Console.WriteLine($"*-----------------------------------------------------------");
Console.WriteLine($" AccuracyMacro = {metrics.MacroAccuracy:0.####}, a value between 0 and 1, the closer to 1, the better");
Console.WriteLine($" AccuracyMicro = {metrics.MicroAccuracy:0.####}, a value between 0 and 1, the closer to 1, the better");
Console.WriteLine($" LogLoss = {metrics.LogLoss:0.####}, the closer to 0, the better");
int i = 0;
foreach (var classLogLoss in metrics.PerClassLogLoss)
{
i++;
Console.WriteLine($" LogLoss for class {i} = {classLogLoss:0.####}, the closer to 0, the better");
}
Console.WriteLine($"************************************************************");
//08 做預測
var predictionEngine = mlContext.Model
.CreatePredictionEngine <InMemoryImageData, ImagePrediction>(trainedModel);
List <InMemoryImageData> imagesToPredict = new List <InMemoryImageData>();
var images_path = Directory.GetFiles(Path.Combine(dataPath, "mask_dataset"));
foreach (var filepath in images_path)
{
var imageToPredict = new InMemoryImageData(File.ReadAllBytes(filepath),
"mask", Path.GetFileName(filepath));
imagesToPredict.Add(imageToPredict);
}
foreach (var img2pre in imagesToPredict)
{
var prediction = predictionEngine.Predict(img2pre);
Console.WriteLine(
$"Image Filename : [{img2pre.ImageFileName}], " +
$"Scores : [{string.Join(",", prediction.Score)}], " +
$"Predicted Label : {prediction.PredictedLabel}");
}
new System.Threading.AutoResetEvent(false).WaitOne();
}
static void Main(string[] args)
{
string assetsRelativePath = @"../../../assets";
string assetsPath = GetAbsolutePath(assetsRelativePath);
string imagesFolderPathForPredictions = Path.Combine(assetsPath, "inputs", "images-for-predictions");
var imageClassifierModelZipFilePath = Path.Combine(assetsPath, "inputs", "MLNETModel", "imageClassifier.zip");
try
{
MLContext mlContext = new MLContext(seed: 1);
Console.WriteLine($"Loading model from: {imageClassifierModelZipFilePath}");
// Load the model
ITransformer loadedModel = mlContext.Model.Load(imageClassifierModelZipFilePath, out var modelInputSchema);
// Create prediction engine to try a single prediction (input = ImageData, output = ImagePrediction)
var predictionEngine = mlContext.Model.CreatePredictionEngine <InMemoryImageData, ImagePrediction>(loadedModel);
//Predict the first image in the folder
IEnumerable <InMemoryImageData> imagesToPredict = LoadInMemoryImagesFromDirectory(
imagesFolderPathForPredictions, false);
InMemoryImageData imageToPredict = new InMemoryImageData
{
Image = imagesToPredict.First().Image,
ImageFileName = imagesToPredict.First().ImageFileName
};
var prediction = predictionEngine.Predict(imageToPredict);
// Get the highest score and its index
float maxScore = prediction.Score.Max();
////////
// Double-check using the index
int maxIndex = prediction.Score.ToList().IndexOf(maxScore);
VBuffer <ReadOnlyMemory <char> > keys = default;
predictionEngine.OutputSchema[4].GetKeyValues(ref keys);
var keysArray = keys.DenseValues().ToArray();
var predictedLabelString = keysArray[maxIndex];
////////
Console.WriteLine($"Image Filename : [{imageToPredict.ImageFileName}], " +
$"Predicted Label : [{prediction.PredictedLabel}], " +
$"Probability : [{maxScore}] "
);
//Predict all images in the folder
//
Console.WriteLine("");
Console.WriteLine("Predicting several images...");
foreach (InMemoryImageData currentImageToPredict in imagesToPredict)
{
var currentPrediction = predictionEngine.Predict(currentImageToPredict);
Console.WriteLine($"Image Filename : [{currentImageToPredict.ImageFileName}], " +
$"Predicted Label : [{currentPrediction.PredictedLabel}], " +
$"Probability : [{currentPrediction.Score.Max()}] "
);
}
//Console.WriteLine("*** Showing all the predictions ***");
//// Find the original label names.
//VBuffer<ReadOnlyMemory<char>> keys = default;
//predictionsDataView.Schema["LabelAsKey"].GetKeyValues(ref keys);
//var originalLabels = keys.DenseValues().ToArray();
//List<ImagePredictionEx> predictions = mlContext.Data.CreateEnumerable<ImagePredictionEx>(predictionsDataView, false, true).ToList();
//predictions.ForEach(pred => ConsoleWriteImagePrediction(pred.ImagePath, pred.Label, (originalLabels[pred.PredictedLabel]).ToString(), pred.Score.Max()));
// OTHER CASE:
// Find the original label names.
//VBuffer<ReadOnlyMemory<char>> keys = default;
//predictionEngine.OutputSchema["LabelAsKey"].GetKeyValues(ref keys);
//var originalLabels = keys.DenseValues().ToArray();
////var index = prediction.PredictedLabel;
//Console.WriteLine($"In-Memory Image provided, " +
// $"Scores : [{string.Join(",", prediction.Score)}], " +
// $"Predicted Label : {prediction.PredictedLabel}");
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
Console.WriteLine("Press any key to end the app..");
Console.ReadKey();
}
