/// <summary>
/// Plays custom objects found in frames of real time video.
/// Each API call takes about 1-2s, therefore we have to introduce a
/// busy lock in order not to overcrowd.
/// This method cant be synchronous since it would block the changes
/// in color of the background of the app.
/// </summary>
/// <param name="image">source image</param>
/// <returns>Nothing</returns>
public async Task PlayObjects(CGImage image)
{
if (busy)
{
return;
}
busy = true;
CGSize size = new CGSize(image.Width, image.Height);
UIGraphics.BeginImageContext(size);
CGRect rect = new CGRect(CGPoint.Empty, size);
UIImage.FromImage(image).Draw(rect);
UIImage ui_image = UIGraphics.GetImageFromCurrentImageContext();
UIGraphics.EndImageContext();
using (Stream stream = ui_image.AsJPEG().AsStream())
{
var result = await endpoint.DetectImageAsync(project_id, published_name, stream);
foreach (var prediction in result.Predictions)
{
if (
supported_items_to_sounds.ContainsKey(prediction.TagName.ToLower()) &&
prediction.Probability > Constants.CONFIDENCE_TRESHOLD)
{
supported_items_to_sounds[prediction.TagName.ToLower()].Play();
break;
}
}
}
busy = false;
}
//public void OnGet()
//{
//}
public async Task <IActionResult> OnPostAsync()
{
// Show image on web page
var imageFilePath = Path.Combine(_host.WebRootPath, "images\\uploadedImage.png");
using (var fileStream = new FileStream(imageFilePath, FileMode.OpenOrCreate))
{
await ImageFile.CopyToAsync(fileStream);
}
ImageFileUrl = "/images/uploadedImage.png";
// Post image to Custom Vision, get resut and show on web page
var cvClient = new CustomVisionPredictionClient()
{
Endpoint = cvEndpoint,
ApiKey = cvPredictionKey
};
try
{
var cvResult = await cvClient.DetectImageAsync(
Guid.Parse(cvProjectId), cvPublishName, ImageFile.OpenReadStream());
if (cvResult.Predictions.Count > 0)
{
Result = "結果:";
Predictions = cvResult.Predictions;
// Draw rectangle on detected image
var detectedImagePath = Path.Combine(_host.WebRootPath, "images\\detectedImage.png");
var rectangleImagePath = Path.Combine(_host.WebRootPath, "images\\rectangle.png");
using (var detectedImage = new Bitmap(imageFilePath))
using (var graphics = Graphics.FromImage(detectedImage))
{
graphics.DrawImage(new Bitmap(rectangleImagePath),
(int)(cvResult.Predictions[0].BoundingBox.Left * detectedImage.Width),
(int)(cvResult.Predictions[0].BoundingBox.Top * detectedImage.Height),
(int)(cvResult.Predictions[0].BoundingBox.Width * detectedImage.Width),
(int)(cvResult.Predictions[0].BoundingBox.Height * detectedImage.Height));
detectedImage.Save(detectedImagePath, ImageFormat.Png);
}
ImageFileUrl = "/images/detectedImage.png";
}
else
{
Result = "判定できませんでした";
}
}
catch (CustomVisionErrorException e)
{
Result = "エラー: " + e.Message;
}
return(Page());
}
private async Task <ImagePrediction> GetPredictionAsync(byte[] imageFile)
{
var predictionClient = new CustomVisionPredictionClient
{
ApiKey = _cvSecrets.PredictionKey,
Endpoint = _cvSecrets.PredictionUrl
};
var stream = new MemoryStream(imageFile);
return(await predictionClient.DetectImageAsync(_cvSecrets.ProjectId, _cvSecrets.PublishedName, stream));
}
async Task PredictPhoto(MediaFile photo)
{
var endpoint = new CustomVisionPredictionClient
{
ApiKey = await KeyService.GetPredictionKey(),
Endpoint = await KeyService.GetEndpoint()
};
var results = await endpoint.DetectImageAsync(Guid.Parse(await KeyService.GetProjectId()),
await KeyService.GetPublishName(),
photo.GetStream());
AllPredictions = results.Predictions
.Where(p => p.Probability > Probability)
.ToList();
}
static public async void ClassifyImage(Guid guid, string path)
{
using (var stream = File.Open(path, FileMode.Open))
{
var imageresult = await endpoint.DetectImageAsync(guid, publishedModelName, stream);
var result = imageresult.Predictions.Select(x => x.Probability).ToList();
var maskresult = result.Max();
if (maskresult >= 0.75)
{
Console.WriteLine("有戴口罩");
}
else
{
Console.WriteLine("沒戴口罩");
}
}
}
private async Task <IEnumerable <PredictionModel> > GetMeaningfulpredictions(byte[] image)
{
using (var stream = new MemoryStream(image))
{
var client = new CustomVisionPredictionClient()
{
ApiKey = _config.PredictionKey,
Endpoint = _config.PredictionEndpoint
};
var predictionResult = await client.DetectImageAsync(new Guid(_config.ProjectId), _config.ModelName, stream);
foreach (var c in predictionResult.Predictions)
{
_log.LogDebug($"\t{c.TagName}: {c.Probability:P1} [ {c.BoundingBox.Left}, {c.BoundingBox.Top}, {c.BoundingBox.Width}, {c.BoundingBox.Height} ]");
}
var meaningfulPredictions = predictionResult.Predictions.Where(x => x.Probability > 0.15);
return(meaningfulPredictions);
}
}
public async Task <string> DetectAsync(IFormFile file, int lightness)
{
var predictionSection = _config.GetSection("PredictionApi");
var client = new CustomVisionPredictionClient
{
ApiKey = predictionSection["key"],
Endpoint = predictionSection["endpoint"]
};
var result = await client.DetectImageAsync(Guid.Parse(predictionSection["projectid"]),
predictionSection["iteration"], file.OpenReadStream());
if (lightness == -1)
{
return(DrawRectangles(file, result));
}
else
{
return(DrawLightRectangles(file, result, lightness));
}
}
public static async System.Threading.Tasks.Task Run(
[BlobTrigger("pending/{name}")] Stream image,
[Queue(Constants.QUEUE_NAME)] IAsyncCollector <string> applicationQueue,
string name,
ILogger log,
ExecutionContext executionContext)
{
var sourceStream = new MemoryStream();
await image.CopyToAsync(sourceStream);
var bitmap = new Bitmap(sourceStream);
var customVisionPredictionClient = new CustomVisionPredictionClient
{
ApiKey = Environment.GetEnvironmentVariable("CustomVisionPredictionClient_ApiKey"),
Endpoint = Environment.GetEnvironmentVariable("CustomVisionPredictionClient_Endpoint")
};
sourceStream.Position = 0;
var response = await customVisionPredictionClient.DetectImageAsync(Guid.Parse(Environment.GetEnvironmentVariable("CustomVisionPredictionClient_ProjectId")), "Completed Route",
sourceStream);
var routes = new List <string>();
foreach (var predictionModel in response.Predictions)
{
if (predictionModel.TagName == "Completed Route" && predictionModel.Probability > 0.85)
{
var cropped = CropBitmap(bitmap,
predictionModel.BoundingBox.Left,
predictionModel.BoundingBox.Top,
predictionModel.BoundingBox.Width,
predictionModel.BoundingBox.Height);
var memoryStream = new MemoryStream();
//ONLY FOR DEBUG
//cropped.Save(Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Desktop),Guid.NewGuid().ToString()));
cropped.Save(memoryStream, System.Drawing.Imaging.ImageFormat.Jpeg);
memoryStream.Position = 0;
//https://stackoverflow.com/questions/53367132/where-to-store-files-for-azure-function
var path = Path.Combine(executionContext.FunctionAppDirectory, "Zwift-5c2367dfe003.json");
Environment.SetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS", path);
Image tmpImage = await Image.FromStreamAsync(memoryStream);
var client = await ImageAnnotatorClient.CreateAsync();
var tmp = await client.DetectTextAsync(tmpImage);
var annotation = tmp.FirstOrDefault();
if (annotation?.Description != null)
{
routes.Add(annotation.Description.Replace("\n", " ").Trim());
}
}
}
if (routes.Count > 0)
{
var user = name.Split("_").First();
await applicationQueue.AddAsync(JsonConvert.SerializeObject(new MultipleRoutesCompletedModel
{
UserId = user,
Routes = routes
}));
await applicationQueue.FlushAsync();
}
}
public static async Task Main(string[] args)
{
// Project name must match the one in Custom Vision
var projectName = "Open Images Detector";
var publishedModelName = "OpenImagesDetectorModel";
// Replace with your own from Custom Vision project
var customVisionKey = "";
var endpoint = "";
var resourceId = "";
if (string.IsNullOrWhiteSpace(endpoint) ||
string.IsNullOrWhiteSpace(customVisionKey) ||
string.IsNullOrWhiteSpace(resourceId))
{
Console.WriteLine("You need to set the endpoint, key and resource id. The program will end;");
Console.ReadKey();
return;
}
// Training Client
var trainingApi = new CustomVisionTrainingClient()
{
ApiKey = customVisionKey,
Endpoint = endpoint
};
Console.WriteLine($"----- Selecting existing project: {projectName}... -----");
var projects = await trainingApi.GetProjectsAsync();
var project = projects.FirstOrDefault(x => x.Name == projectName);
if (project != null)
{
Console.WriteLine($"\t{projectName} found in Custom Vision workspace.");
var domainId = project.Settings.DomainId;
var projectDomain = await trainingApi.GetDomainAsync(domainId);
Console.WriteLine($"\tProject domain: {projectDomain.Name}.");
Console.WriteLine($"\tProject type: {projectDomain.Type}.");
WriteSeparator();
}
else
{
Console.WriteLine($"\tProject {projectName} was not found in your subscription. The program will end.");
return;
}
Console.WriteLine("Press any key to continue...");
Console.ReadKey();
// Retrieve the tags that already exist in the project
Console.WriteLine("----- Retrieving tags... -----");
var modelTags = await trainingApi.GetTagsAsync(project.Id);
var tags = new List <Tag>();
var onlineImages = 0;
// Obtain tags from our dataset
var tagsFile = Path.Combine(trainingFolder, "tags.txt");
var imageLabels = await File.ReadAllLinesAsync(tagsFile);
foreach (var label in imageLabels)
{
// Check if the label already exists
var tag = modelTags.FirstOrDefault(x => x.Name == label);
if (tag == null)
{
// If not, create it
tag = await trainingApi.CreateTagAsync(project.Id, label);
Console.WriteLine($"\tTag {tag.Name} was created.");
}
else
{
// If so, just count images with this tag
onlineImages += tag.ImageCount;
Console.WriteLine($"\tTag {label} was NOT created (it already exists)");
}
tags.Add(tag);
}
WriteSeparator();
Console.WriteLine("Press any key to continue...");
Console.ReadKey();
// Upload images
var uploadImages = true;
if (onlineImages > 0)
{
Console.WriteLine($"There are {onlineImages} training images already uploaded. Do you want to upload more? (Y/N)");
uploadImages = Console.ReadKey().Key == ConsoleKey.Y;
}
Iteration iteration = null;
if (uploadImages)
{
Console.WriteLine("----- Accessing images -----");
var imageFileEntries = new List <ImageFileCreateEntry>();
foreach (var label in imageLabels)
{
var tagFolder = Path.Combine(trainingFolder, label);
var tagImages = Directory.GetFiles(tagFolder);
var tagLabels = Path.Combine(tagFolder, "normalizedLabel");
foreach (var image in tagImages)
{
var imageName = Path.GetFileNameWithoutExtension(image);
var imageFile = Path.GetFileName(image);
var imageLabelFile = $"{imageName}.txt";
var tagCV = tags.Single(x => x.Name == label);
var imageBoundingBoxes = await File.ReadAllLinesAsync(Path.Combine(tagLabels, imageLabelFile));
var imageRegions = new List <Region>();
foreach (var bbox in imageBoundingBoxes)
{
var normalizedData = bbox.Split();
var left = float.Parse(normalizedData[0]);
var top = float.Parse(normalizedData[1]);
var width = float.Parse(normalizedData[2]);
var height = float.Parse(normalizedData[3]);
imageRegions.Add(new Region(tagCV.Id, left, top, width, height));
}
Console.WriteLine($"\tAdding image {imageName} with its regions.");
imageFileEntries.Add(new ImageFileCreateEntry(
imageFile, await File.ReadAllBytesAsync(image), null, imageRegions));
}
}
var batchPageSize = 64;
var numBatches = imageFileEntries.Count / batchPageSize;
var sizeLastBatch = imageFileEntries.Count % batchPageSize;
for (int batch = 0; batch < numBatches; batch++)
{
Console.WriteLine($"\tUploading images batch #{batch}.");
var entries = imageFileEntries.Skip(batch * batchPageSize).Take(batchPageSize).ToList();
await trainingApi.CreateImagesFromFilesAsync(
project.Id, new ImageFileCreateBatch(entries));
}
if (sizeLastBatch > 0)
{
Console.WriteLine($"\tUploading last batch.");
var lastEntries = imageFileEntries.Skip(numBatches * batchPageSize).Take(sizeLastBatch).ToList();
await trainingApi.CreateImagesFromFilesAsync(project.Id,
new ImageFileCreateBatch(lastEntries));
}
WriteSeparator();
Console.WriteLine("Press any key to continue...");
Console.ReadKey();
try
{
// Now there are images with tags start training the project
Console.WriteLine("----- Starting the Training process... -----");
iteration = await trainingApi.TrainProjectAsync(project.Id);
// The returned iteration will be in progress, and can be queried periodically to see when it has completed
while (iteration.Status == "Training")
{
Thread.Sleep(1000);
Console.WriteLine($"\tIteration '{iteration.Name}' status: {iteration.Status}");
// Re-query the iteration to get it's updated status
iteration = await trainingApi.GetIterationAsync(project.Id, iteration.Id);
}
Console.WriteLine($"\tIteration '{iteration.Name}' status: {iteration.Status}");
WriteSeparator();
Console.WriteLine("Press any key to continue...");
Console.ReadKey();
// The iteration is now trained. Publish it to the prediction endpoint.
Console.WriteLine($"----- Starting the Publication process. -----");
await trainingApi.PublishIterationAsync(
project.Id, iteration.Id, publishedModelName, resourceId);
Console.WriteLine($"\tIteration '{iteration.Name}' published.");
WriteSeparator();
Console.WriteLine("Press any key to continue...");
Console.ReadKey();
}
catch (Exception ex)
{
Console.WriteLine($"There was an exception (perhaps nothing changed since last iteration?).");
}
}
if (iteration == null)
{
var iterations = await trainingApi.GetIterationsAsync(project.Id);
iteration = iterations.OrderByDescending(x => x.LastModified).FirstOrDefault();
Console.WriteLine($"Iteration '{iteration.Name}' found and loaded.");
WriteSeparator();
Console.WriteLine("Press any key to continue...");
Console.ReadKey();
}
// Prediction Client
var predictionClient = new CustomVisionPredictionClient()
{
ApiKey = customVisionKey,
Endpoint = endpoint
};
// Make predictions against the new project
Console.WriteLine("----- Making predictions -----");
var testImages = LoadImagesFromDisk("Test");
foreach (var image in testImages)
{
var imageName = Path.GetFileName(image);
using (var stream = new MemoryStream(File.ReadAllBytes(image)))
{
Console.WriteLine($"\tImage: {imageName}");
var result = await predictionClient.DetectImageAsync(
project.Id, publishedModelName, stream);
// Loop over each prediction and write out the results
foreach (var prediction in result.Predictions.OrderByDescending(x => x.Probability))
{
Console.WriteLine($"\t\tFor Tag '{prediction.TagName}': {prediction.Probability:P3} " +
$"[{prediction.BoundingBox.Left}, {prediction.BoundingBox.Top}, " +
$"{prediction.BoundingBox.Width}, {prediction.BoundingBox.Height}]");
}
WriteSeparator();
Console.WriteLine("Press any key for next image...");
Console.ReadKey();
}
}
WriteSeparator();
Console.WriteLine("----- Do you want to export the model? (Y/N) -----");
var exportModel = Console.ReadKey().Key == ConsoleKey.Y;
if (exportModel)
{
do
{
var platform = string.Empty;
var extension = string.Empty;
Export export;
Console.WriteLine("\tOptions: \n\t1) TensorFlow \n\t2) CoreML \n\t3) Other platform \n\tE) End program");
var option = Console.ReadKey().Key;
switch (option)
{
case ConsoleKey.D1:
platform = "TensorFlow";
extension = "zip";
break;
case ConsoleKey.D2:
platform = "CoreML";
extension = "mlmodel";
break;
case ConsoleKey.D3:
Console.WriteLine("\tType the platform name");
platform = Console.ReadLine();
Console.WriteLine($"\tNow type the file extension for the {platform} exported model.");
extension = Console.ReadLine();
break;
case ConsoleKey.E:
exportModel = false;
break;
default:
Console.WriteLine("\n\tOption not supported.");
break;
}
WriteSeparator();
if (!string.IsNullOrWhiteSpace(platform))
{
try
{
Console.WriteLine($"\tExporting to {platform}...");
do
{
var exports = await trainingApi.GetExportsAsync(project.Id, iteration.Id);
export = exports.FirstOrDefault(x => x.Platform == platform);
if (export == null)
{
export = await trainingApi.ExportIterationAsync(project.Id, iteration.Id, platform);
}
Thread.Sleep(1000);
Console.WriteLine($"\tStatus: {export.Status}");
} while (export.Status == "Exporting");
Console.WriteLine($"Status: {export.Status}");
if (export.Status == ExportStatus.Done)
{
Console.WriteLine($"\tDownloading {platform} model");
var filePath = Path.Combine(Environment.CurrentDirectory, $"{publishedModelName}_{platform}.{extension}");
using (var httpClient = new HttpClient())
{
using (var stream = await httpClient.GetStreamAsync(export.DownloadUri))
{
using (var file = new FileStream(filePath, FileMode.Create))
{
await stream.CopyToAsync(file);
Console.WriteLine($"\tModel successfully exported. You can find it here:\n\t{filePath}.");
WriteSeparator();
}
}
}
}
}
catch (Exception ex)
{
Console.WriteLine($"Exception found: {ex.Message}");
WriteSeparator();
}
}
} while (exportModel);
}
Console.WriteLine("Press a key to exit the program!");
Console.ReadKey();
}
public static async Task Predict()
{
// Use test images in this repo - substitute other as needed. Note, the API also takes URLs to pictures, in this case I am sending picture as request payload.
List <string> filePaths = Directory.GetFiles(@"..\..\..\..\..\Images\test\", "*.*", SearchOption.TopDirectoryOnly).ToList();
StringBuilder sb = new StringBuilder();
using (CustomVisionPredictionClient endpoint = new CustomVisionPredictionClient()
{
ApiKey = _predictionKey, Endpoint = _apiEndpoint
})
{
foreach (string filePath in filePaths)
{
sb.AppendLine(Path.GetFileName(filePath));
sb.AppendLine(DateTime.UtcNow.ToString());
using (var stream = File.OpenRead(filePath))
{
ImagePrediction classificationResult = await endpoint.ClassifyImageAsync(_projectGuid_Classification_Multiclass, _modelName_Classification_Multiclass, stream);
// Loop over each classification and write out the results
foreach (var c in classificationResult.Predictions.OrderByDescending(p => p.Probability))
{
sb.AppendLine($"Classification (multi-class): {c.TagName}: {c.Probability:P1}");
}
}
sb.AppendLine("--------------------");
using (var stream = File.OpenRead(filePath))
{
ImagePrediction classificationResult = await endpoint.ClassifyImageAsync(_projectGuid_Classification_Multilabel, _modelName_Classification_Multilabel, stream);
// Loop over each classification and write out the results
foreach (var c in classificationResult.Predictions.OrderByDescending(p => p.Probability))
{
sb.AppendLine($"Classification (multi-label): {c.TagName}: {c.Probability:P1}");
}
}
sb.AppendLine("--------------------");
using (var stream = File.OpenRead(filePath))
{
ImagePrediction detectionResult = await endpoint.DetectImageAsync(_projectGuid_Detection, _modelName_Detection, stream);
// Loop over each detection and write out the results
foreach (var c in detectionResult.Predictions.OrderByDescending(p => p.Probability))
{
sb.AppendLine($"Detection: {c.TagName}: {c.Probability:P1} " + $"[ {c.BoundingBox.Left}, {c.BoundingBox.Top}, {c.BoundingBox.Width}, {c.BoundingBox.Height} ]");
}
}
sb.AppendLine("--------------------");
sb.AppendLine();
}
}
string final = sb.ToString();
File.WriteAllText(_resultsFilePath, final);
}