public IActionResult Get([FromQuery] string url)
{
string imageFileRelativePath = @"../../../assets" + url;
string imageFilePath = CommonHelpers.GetAbsolutePath(imageFileRelativePath);
try
{
Image image = Image.FromFile(imageFilePath);
//Convert to Bitmap
Bitmap bitmapImage = (Bitmap)image;
//Set the specific image data into the ImageInputData type used in the DataView
ImageInputData imageInputData = new ImageInputData {
Image = bitmapImage
};
//Detect the objects in the image
var result = DetectAndPaintImage(imageInputData, imageFilePath);
return(Ok(result));
}
catch (Exception e)
{
_logger.LogInformation("Error is: " + e.Message);
return(BadRequest());
}
}
public void DetectObjectsUsingModel(ImageInputData imageInputData)
{
var probs = model.Predict(imageInputData).PredictedLabels;
IList <YoloBoundingBox> boundingBoxes = _parser.ParseOutputs(probs);
filteredBoxes = _parser.FilterBoundingBoxes(boundingBoxes, 5, .5F);
}
public void DetectObjectsUsingModel(ImageInputData imageInputData)
{
var probs = predictionEngine.Predict(imageInputData).PredictedLabels;
List <BoundingBox> boundingBoxes = outputParser.ParseOutputs(probs);
filteredBoxes = outputParser.FilterBoundingBoxes(boundingBoxes, 5, .5F);
}
public ImageClassificationResult Classify(MemoryStream image)
{
// TODO: the classification depends on System.Drawing which needs some libraries to be installed in mac/linux
// There are alternatives to Bitmap, like ImageSharp, but would that work with ML.NET ????
// https://www.hanselman.com/blog/HowDoYouUseSystemDrawingInNETCore.aspx
// Can install with: brew install mono-libgdiplus
// Convert to Bitmap
Bitmap bitmapImage = (Bitmap)Image.FromStream(image);
// Set the specific image data into the ImageInputData type used in the DataView
ImageInputData imageInputData = new ImageInputData {
Image = bitmapImage
};
// Predict code for provided image
ImageLabelPredictions imageLabelPredictions = _predictionEnginePool.Predict(imageInputData);
// Predict the image's label (The one with highest probability)
float[] probabilities = imageLabelPredictions.PredictedLabels;
var maxProbability = probabilities.Max();
var maxProbabilityIndex = probabilities.AsSpan().IndexOf(maxProbability);
return(new ImageClassificationResult()
{
Label = _labels[maxProbabilityIndex],
Probability = maxProbability
});
}
/// <summary>
/// Gets bounding boxes/objects from bitmap
/// </summary>
/// <param name="imageInputData"></param>
/// <returns></returns>
private List <BoundingBox> GetObjectsFromModel(ImageInputData imageInputData)
{
var labels = CustomVisionPredictionEngine?.Predict(imageInputData).PredictedLabels ?? TinyYoloPredictionEngine?.Predict(imageInputData).PredictedLabels;
var boundingBoxes = OutputParser.ParseOutputs(labels);
var filteredBoxes = OutputParser.FilterBoundingBoxes(boundingBoxes, 5, 0.5f);
return(filteredBoxes);
}
public List <BoundingBox> DetectObjectsUsingModel(ImageInputData imageInputData)
{
var labels = customVisionPredictionEngine?.Predict(imageInputData).PredictedLabels ?? tinyYoloPredictionEngine?.Predict(imageInputData).PredictedLabels;
var boundingBoxes = outputParser.ParseOutputs(labels);
var filteredBoxes = outputParser.FilterBoundingBoxes(boundingBoxes, 5, 0.5f);
return(filteredBoxes);
}
private List <BoundingBox> DetectObjectsUsingModel(ImageInputData imageInputData)
{
var labels = fasterRCNNPredictionEngine?.Predict(imageInputData).PredictedLabels ?? fasterRCNNPredictionEngine?.Predict(imageInputData).PredictedLabels;
var boundingBoxes = outputParser.ParseOutputs(labels);
var filteredBoxes = outputParser.FilterBoundingBoxes(boundingBoxes, 5, 0.5f);
return(filteredBoxes);
}
private ImagePredictedLabelWithProbability PredictDataUsingModel(ImageInputData imageName)
{
var image1Probabilities = _model.Predict(imageName).PredictedLabels;
var bestLabelPrediction = new ImagePredictedLabelWithProbability();
(bestLabelPrediction.PredictedLabel, bestLabelPrediction.Probability) = ModelHelpers.GetBestLabel(_labels, image1Probabilities);
return(bestLabelPrediction);
}
public ImagePredictedLabelWithProbability Score(ImageInputData imageName)
{
if (_model == null)
{
Init();
}
var predictions = PredictDataUsingModel(imageName);
return(predictions);
}
public async Task <IActionResult> IdentifyObjects(IFormFile imageFile)
{
if (imageFile.Length == 0)
{
return(BadRequest());
}
try
{
MemoryStream imageMemoryStream = new MemoryStream();
await imageFile.CopyToAsync(imageMemoryStream);
//Check that the image is valid
byte[] imageData = imageMemoryStream.ToArray();
if (!imageData.IsValidImage())
{
return(StatusCode(StatusCodes.Status415UnsupportedMediaType));
}
//Convert to Image
Image image = Image.FromStream(imageMemoryStream);
string fileName = string.Format("{0}.Jpeg", image.GetHashCode());
string imageFilePath = Path.Combine(_imagesTmpFolder, fileName);
//save image to a path
image.Save(imageFilePath, ImageFormat.Jpeg);
//Convert to Bitmap
Bitmap bitmapImage = (Bitmap)image;
_logger.LogInformation($"Start processing image...");
//Measure execution time
var watch = System.Diagnostics.Stopwatch.StartNew();
//Set the specific image data into the ImageInputData type used in the DataView
ImageInputData imageInputData = new ImageInputData {
Image = bitmapImage
};
//Detect the objects in the image
var result = DetectAndPaintImage(imageInputData, imageFilePath);
//Stop measuring time
watch.Stop();
var elapsedMs = watch.ElapsedMilliseconds;
_logger.LogInformation($"Image processed in {elapsedMs} miliseconds");
return(Ok(result));
}
catch (Exception e)
{
_logger.LogInformation("Error is: " + e.Message);
return(BadRequest());
}
}
public async Task IdentifyObjects(IFormFile imageFile, Guid id)
{
try
{
MemoryStream imageMemoryStream = new MemoryStream();
await imageFile.CopyToAsync(imageMemoryStream);
//Check that the image is valid
byte[] imageData = imageMemoryStream.ToArray();
//Convert to Image
Image image = Image.FromStream(imageMemoryStream);
string fileName = string.Format("{0}.Jpeg", image.GetHashCode());
string imageFilePath = Path.Combine(_imagesTmpFolder, fileName);
////
bool exists = System.IO.Directory.Exists(_imagesTmpFolder);
if (!exists)
{
System.IO.Directory.CreateDirectory(_imagesTmpFolder);
}
//save image to a path
image.Save(imageFilePath, ImageFormat.Jpeg);
///
//Convert to Bitmap
Bitmap bitmapImage = (Bitmap)image;
//Measure execution time
var watch = System.Diagnostics.Stopwatch.StartNew();
//Set the specific image data into the ImageInputData type used in the DataView
ImageInputData imageInputData = new ImageInputData {
Image = bitmapImage
};
//Detect the objects in the image
var result = DetectAndPaintImage(imageInputData, imageFilePath);
//Stop measuring time
watch.Stop();
elapsedMs = watch.ElapsedMilliseconds;
result.ElapsedTime = elapsedMs;
result.imageStringOriginal = imageData;
await _imageService.AddImage(id, result);
}
catch (Exception e)
{
throw new Exception(e.Message);
}
}
protected IEnumerable <ImagePredictedLabelWithProbability> PredictDataUsingModel(string testLocation, string imagesFolder, string labelsLocation, PredictionFunction <ImageInputData, ImageNetPrediction> model)
{
ConsoleWriteHeader("Classificate images");
Console.WriteLine($"Images folder: {imagesFolder}");
Console.WriteLine($"Training file: {testLocation}");
Console.WriteLine($"Labels file: {labelsLocation}");
var labels = ModelHelpers.ReadLabels(labelsLocation);
/////////////////////////////////////////////////////////////////////////////////////
// IMAGE 1
// Predict label for "green-office-chair-test.jpg"
var image1 = new ImageInputData {
ImagePath = imagesFolder + "\\" + "green-office-chair-test.jpg"
};
var image1Probabilities = model.Predict(image1).PredictedLabels;
//Set a single label as predicted or even none if probabilities were lower than 70%
var image1BestLabelPrediction = new ImagePredictedLabelWithProbability()
{
ImagePath = image1.ImagePath,
};
(image1BestLabelPrediction.PredictedLabel, image1BestLabelPrediction.Probability) = GetBestLabel(labels, image1Probabilities);
image1BestLabelPrediction.ConsoleWrite();
yield return(image1BestLabelPrediction);
/////////////////////////////////////////////////////////////////////////////////////
// IMAGE 2
// Predict label for "high-metal-office-chair.jpg"
var image2 = new ImageInputData {
ImagePath = imagesFolder + "\\" + "high-metal-office-chair.jpg"
};
var image2Probabilities = model.Predict(image2).PredictedLabels;
//Set a single label as predicted or even none if probabilities were lower than 70%
var image2BestLabelPrediction = new ImagePredictedLabelWithProbability()
{
ImagePath = image2.ImagePath,
};
(image2BestLabelPrediction.PredictedLabel, image2BestLabelPrediction.Probability) = GetBestLabel(labels, image2Probabilities);
image2BestLabelPrediction.ConsoleWrite();
yield return(image1BestLabelPrediction);
}
public async Task <IActionResult> ClassifyImage(IFormFile imageFile)
{
if (imageFile.Length == 0)
{
return(BadRequest());
}
var imageMemoryStream = new MemoryStream();
await imageFile.CopyToAsync(imageMemoryStream);
// Check that the image is valid.
byte[] imageData = imageMemoryStream.ToArray();
if (!imageData.IsValidImage())
{
return(StatusCode(StatusCodes.Status415UnsupportedMediaType));
}
// Convert to Image.
Image image = Image.FromStream(imageMemoryStream);
// Convert to Bitmap.
Bitmap bitmapImage = (Bitmap)image;
_logger.LogInformation("Start processing image...");
// Measure execution time.
var watch = System.Diagnostics.Stopwatch.StartNew();
// Set the specific image data into the ImageInputData type used in the DataView.
var imageInputData = new ImageInputData {
Image = bitmapImage
};
// Predict code for provided image.
ImageLabelPredictions imageLabelPredictions = _predictionEnginePool.Predict(imageInputData);
// Stop measuring time.
watch.Stop();
var elapsedMs = watch.ElapsedMilliseconds;
_logger.LogInformation($"Image processed in {elapsedMs} miliseconds");
// Predict the image's label (The one with highest probability).
ImagePredictedLabelWithProbability imageBestLabelPrediction
= FindBestLabelWithProbability(imageLabelPredictions, imageInputData);
return(Ok(imageBestLabelPrediction));
}
/// <summary>
/// Parent method of <see cref="GetObjectsFromModel" /> and <see cref="DrawOverlays" />
/// </summary>
/// <param name="bitmap"></param>
/// <param name="token"></param>
/// <returns></returns>
private async Task DrawOverlaysFromBitmapDetectedObjects(Bitmap bitmap, CancellationToken token)
{
var frame = new ImageInputData {
Image = bitmap
};
var boundingBoxes = GetObjectsFromModel(frame);
if (!token.IsCancellationRequested)
{
await Application.Current.Dispatcher.InvokeAsync(() =>
{
VideoImage.Source = bitmap.ToBitmapImage();
DrawOverlays(bitmap, boundingBoxes);
});
}
}
private async Task <List <BoundingBox> > ParseDroneCameraFrame(Bitmap stream, int width, int height)
{
if (fasterRCNNPredictionEngine == null)
{
return(null);
}
var frame = new ImageInputData {
Image = stream
};
var filteredBoxes = DetectObjectsUsingModel(frame);
return(filteredBoxes);
}
private ImagePredictedLabelWithProbability PredictDataUsingModel(string imageName)
{
var inputImage = new ImageInputData {
ImagePath = imageName
};
var image1Probabilities = _model.Predict(inputImage).PredictedLabels;
//Set a single label as predicted or even none if probabilities were lower than 70%
var bestLabelPrediction = new ImagePredictedLabelWithProbability()
{
ImagePath = inputImage.ImagePath,
};
(bestLabelPrediction.PredictedLabel, bestLabelPrediction.Probability) = ModelHelpers.GetBestLabel(_labels, image1Probabilities);
return(bestLabelPrediction);
}
static void Main(string[] args)
{
var modelpath = Path.Combine(Environment.CurrentDirectory, "MLModel", "mlmodel.zip");
var imgspath = Path.Combine(Environment.CurrentDirectory, "ImgDatas");
var labels = System.IO.File.ReadAllLines(Path.Combine(Environment.CurrentDirectory, "MLModel", "labels.txt"));
MLContext mlContext = new MLContext(seed: 1);
// Load the model
ITransformer loadedModel = mlContext.Model.Load(modelpath, out var schema);
// Make prediction (input = ImageNetData, output = ImageNetPrediction)
var predictor = mlContext.Model.CreatePredictionEngine <ImageInputData, ImageLabelPrediction>(loadedModel);
DirectoryInfo imgdir = new DirectoryInfo(imgspath);
foreach (var jpgfile in imgdir.GetFiles("*.jpg"))
{
var imputimg = new ImageInputData {
Image = ConvertToBitmap(jpgfile.FullName)
};
var pred = predictor.Predict(imputimg);
var imgpredresult = new ImagePredictedLabelWithProbability()
{
Name = jpgfile.Name
};
Console.WriteLine($"Filename:{imgpredresult.Name}");
foreach (var item in pred.PredictedLabels)
{
Console.WriteLine($"Predict Result:{item}");
}
var maxresult = pred.PredictedLabels.Max();
if (maxresult >= 0.7)
{
Console.WriteLine($"Predict Label Result:{labels[pred.PredictedLabels.AsSpan().IndexOf(maxresult)]}");
}
Console.WriteLine("===== next image ======");
}
}
private Result DetectAndPaintImage(ImageInputData imageInputData, string imageFilePath)
{
//Predict the objects in the image
_objectDetectionService.DetectObjectsUsingModel(imageInputData);
var img = _objectDetectionService.DrawBoundingBox(imageFilePath);
using (MemoryStream m = new MemoryStream())
{
img.Image.Save(m, img.Image.RawFormat);
byte[] imageBytes = m.ToArray();
var result = new Result {
imageStringProcessed = imageBytes, Description = img.Description, ElapsedTime = elapsedMs
};
return(result);
}
}
public async Task<IActionResult> Post(IFormFile file)
{
if (file == null || file.Length == 0)
{
return BadRequest();
}
var fileName = $"{Guid.NewGuid()}_{ContentDispositionHeaderValue.Parse(file.ContentDisposition).FileName.Trim('"')}";
var fullPath = Path.Combine(_scoringSvc.ImagesFolder, fileName);
try
{
MemoryStream imageMemoryStream = new MemoryStream();
await file.CopyToAsync(imageMemoryStream);
//Convert to Bitmap
Bitmap bitmapImage = (Bitmap)Image.FromStream(imageMemoryStream);
//Set the specific image data into the ImageInputData type used in the DataView
ImageInputData imageInputData = new ImageInputData { Image = bitmapImage };
_logger.LogInformation($"Start processing image file { fullPath }");
var scoring = _scoringSvc.Score(imageInputData);
_logger.LogInformation($"Image processed");
return Ok(ClassificationResponse.CreateFrom(scoring));
}
finally
{
try
{
_logger.LogInformation($"Deleting Image {fullPath}");
System.IO.File.Delete(fullPath);
}
catch (Exception)
{
_logger.LogInformation("Error deleting image: " + fileName);
}
}
}
private Result DetectAndPaintImage(ImageInputData imageInputData, string imageFilePath)
{
//Predict the objects in the image
_objectDetectionService.DetectObjectsUsingModel(imageInputData);
var img = _objectDetectionService.DrawBoundingBox(imageFilePath);
using (MemoryStream m = new MemoryStream())
{
img.Save(m, img.RawFormat);
byte[] imageBytes = m.ToArray();
// Convert byte[] to Base64 String
base64String = Convert.ToBase64String(imageBytes);
var result = new Result {
imageString = base64String
};
return(result);
}
}
async Task ParseWebCamFrame(Bitmap bitmap, CancellationToken token)
{
if (customVisionPredictionEngine == null && tinyYoloPredictionEngine == null)
{
return;
}
var frame = new ImageInputData {
Image = bitmap
};
var filteredBoxes = DetectObjectsUsingModel(frame);
if (!token.IsCancellationRequested)
{
await Application.Current.Dispatcher.InvokeAsync(() =>
{
DrawOverlays(filteredBoxes, WebCamImage.ActualHeight, WebCamImage.ActualWidth);
});
}
}
public ImageClassificationResult Classify(MemoryStream image)
{
// Convert to image to Bitmap and load into an ImageInputData
Bitmap bitmapImage = (Bitmap)Image.FromStream(image);
ImageInputData imageInputData = new ImageInputData {
Image = bitmapImage
};
// Run the model
var imageLabelPredictions = _predictionEnginePool.Predict(imageInputData);
// Find the label with the highest probability
// and return the ImageClassificationResult instance
float[] probabilities = imageLabelPredictions.PredictedLabels;
var maxProbability = probabilities.Max();
var maxProbabilityIndex = probabilities.AsSpan().IndexOf(maxProbability);
return(new ImageClassificationResult()
{
Label = _labels[maxProbabilityIndex],
Probability = maxProbability
});
}
private ImagePredictedLabelWithProbability FindBestLabelWithProbability(ImageLabelPredictions imageLabelPredictions, ImageInputData imageInputData)
{
// Read TF model's labels (labels.txt) to classify the image across those labels.
var labels = ReadLabels(_labelsFilePath);
float[] probabilities = imageLabelPredictions.PredictedLabels;
// Set a single label as predicted or even none if probabilities were lower than 70%.
var imageBestLabelPrediction = new ImagePredictedLabelWithProbability()
{
ImageId = imageInputData.GetHashCode().ToString(), //This ID is not really needed, it could come from the application itself, etc.
};
(imageBestLabelPrediction.PredictedLabel, imageBestLabelPrediction.Probability) = GetBestLabel(labels, probabilities);
return(imageBestLabelPrediction);
}
public void Run()
{
// Create new model context
var mlContext = new MLContext();
// Define the model pipeline:
// 1. loading and resizing the image
// 2. extracting image pixels
// 3. running pre-trained TensorFlow model
var pipeline = mlContext.Transforms.ResizeImages(
outputColumnName: "input",
imageWidth: 224,
imageHeight: 224,
inputColumnName: nameof(ImageInputData.Image)
)
.Append(mlContext.Transforms.ExtractPixels(
outputColumnName: "input",
interleavePixelColors: true,
offsetImage: 117)
)
.Append(mlContext.Model.LoadTensorFlowModel(_tensorFlowModelFilePath)
.ScoreTensorFlowModel(
outputColumnNames: new[] { "softmax2" },
inputColumnNames: new[] { "input" },
addBatchDimensionInput: true));
// Train the model
// Since we are simply using a pre-trained TensorFlow model,
// we can "train" it against an empty dataset
var emptyTrainingSet = mlContext.Data.LoadFromEnumerable(new List <ImageInputData>());
ITransformer mlModel = pipeline.Fit(emptyTrainingSet);
// Save/persist the model to a .ZIP file
// This will be loaded into a PredictionEnginePool by the
// Blazor application, so it can classify new images
mlContext.Model.Save(mlModel, null, _mlnetOutputZipFilePath);
// Create some predictions from output file zip,
// loading the model from the zip file
DataViewSchema predictionPipelineSchema;
mlModel = mlContext.Model.Load(_mlnetOutputZipFilePath, out predictionPipelineSchema);
// create a prediction for classifying one image at a time, taking input data, model, output
var predictionEngine = mlContext.Model.CreatePredictionEngine <ImageInputData, ImageLabelPredictions>(mlModel);
// loading images for some tests
var image = (Bitmap)Bitmap.FromFile("../../SampleImages/toaster.jpg");
var input = new ImageInputData {
Image = image
};
var prediction = predictionEngine.Predict(input);
// get the labels of prediction that showing probability
var maxProbability = prediction.PredictedLabels.Max();
var labelIndex = prediction.PredictedLabels.AsSpan().IndexOf(maxProbability);
var allLabels = System.IO.File.ReadAllLines("TFInceptionModel/imagenet_comp_graph_label_strings.txt");
var classifiedLabel = allLabels[labelIndex];
Console.WriteLine($"Test input image 'toaster.jpg' predicted as '{classifiedLabel}' with probability {100 * maxProbability}%");
}
private ImagePredictedLabelWithProbability FindBestLabelWithProbability(ImageLabelPredictions imageLabelPredictions, ImageInputData imageInputData)
{
//Read TF model's labels (labels.txt) to classify the image across those labels
var labels = ReadLabels(_labelsFilePath);
float[] probabilities = imageLabelPredictions.PredictedLabels;
//Set a single label as predicted or even none if probabilities were lower than 70%
var imageBestLabelPrediction = new ImagePredictedLabelWithProbability()
{
ImagePath = imageInputData.ImagePath,
};
(imageBestLabelPrediction.PredictedLabel, imageBestLabelPrediction.Probability) = GetBestLabel(labels, probabilities);
return(imageBestLabelPrediction);
}
public async Task <IActionResult> ClassifyImage(IFormFile imageFile)
{
if (imageFile.Length == 0)
{
return(BadRequest());
}
//Save the temp image file into the temp-folder
IImageFileWriter imageWriter = new ImageFileWriter();
string fileName = await imageWriter.UploadImageAsync(imageFile, _tempImagesFolderPath);
string imageFilePath = Path.Combine(_tempImagesFolderPath, fileName);
//Convert image stream to byte[]
//
//byte[] imageData = null;
//MemoryStream image = new MemoryStream();
//await imageFile.CopyToAsync(image);
//imageData = image.ToArray();
//if (!imageData.IsValidImage())
// return StatusCode(StatusCodes.Status415UnsupportedMediaType);
_logger.LogInformation($"Start processing image...");
//Measure execution time
var watch = System.Diagnostics.Stopwatch.StartNew();
//Set the specific image data
var imageInputData = new ImageInputData {
ImagePath = imageFilePath
};
//Predict code for provided image
ImageLabelPredictions imageLabelPredictions = _predictionEnginePool.Predict(imageInputData);
//Stop measuring time
watch.Stop();
var elapsedMs = watch.ElapsedMilliseconds;
_logger.LogInformation($"Image processed in {elapsedMs} miliseconds");
//try
//{
// // DELETE FILE WHEN CLOSED
// using (FileStream fs = new FileStream(imageFilePath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.DeleteOnClose))
// {
// // temp file exists
// fs.Close();
// }
//}
//catch (Exception e)
//{
// throw e;
//}
//Predict the image's label (The one with highest probability)
ImagePredictedLabelWithProbability imageBestLabelPrediction
= FindBestLabelWithProbability(imageLabelPredictions, imageInputData);
//return new ObjectResult(result);
return(Ok(imageBestLabelPrediction));
}
