/// <summary> /// Plays custom objects found in frames of real time video. /// Each API call takes about 1-2s, therefore we have to introduce a /// busy lock in order not to overcrowd. /// This method cant be synchronous since it would block the changes /// in color of the background of the app. /// </summary> /// <param name="image">source image</param> /// <returns>Nothing</returns> public async Task PlayObjects(CGImage image) { if (busy) { return; } busy = true; CGSize size = new CGSize(image.Width, image.Height); UIGraphics.BeginImageContext(size); CGRect rect = new CGRect(CGPoint.Empty, size); UIImage.FromImage(image).Draw(rect); UIImage ui_image = UIGraphics.GetImageFromCurrentImageContext(); UIGraphics.EndImageContext(); using (Stream stream = ui_image.AsJPEG().AsStream()) { var result = await endpoint.DetectImageAsync(project_id, published_name, stream); foreach (var prediction in result.Predictions) { if ( supported_items_to_sounds.ContainsKey(prediction.TagName.ToLower()) && prediction.Probability > Constants.CONFIDENCE_TRESHOLD) { supported_items_to_sounds[prediction.TagName.ToLower()].Play(); break; } } } busy = false; }
//public void OnGet() //{ //} public async Task <IActionResult> OnPostAsync() { // Show image on web page var imageFilePath = Path.Combine(_host.WebRootPath, "images\\uploadedImage.png"); using (var fileStream = new FileStream(imageFilePath, FileMode.OpenOrCreate)) { await ImageFile.CopyToAsync(fileStream); } ImageFileUrl = "/images/uploadedImage.png"; // Post image to Custom Vision, get resut and show on web page var cvClient = new CustomVisionPredictionClient() { Endpoint = cvEndpoint, ApiKey = cvPredictionKey }; try { var cvResult = await cvClient.DetectImageAsync( Guid.Parse(cvProjectId), cvPublishName, ImageFile.OpenReadStream()); if (cvResult.Predictions.Count > 0) { Result = "結果:"; Predictions = cvResult.Predictions; // Draw rectangle on detected image var detectedImagePath = Path.Combine(_host.WebRootPath, "images\\detectedImage.png"); var rectangleImagePath = Path.Combine(_host.WebRootPath, "images\\rectangle.png"); using (var detectedImage = new Bitmap(imageFilePath)) using (var graphics = Graphics.FromImage(detectedImage)) { graphics.DrawImage(new Bitmap(rectangleImagePath), (int)(cvResult.Predictions[0].BoundingBox.Left * detectedImage.Width), (int)(cvResult.Predictions[0].BoundingBox.Top * detectedImage.Height), (int)(cvResult.Predictions[0].BoundingBox.Width * detectedImage.Width), (int)(cvResult.Predictions[0].BoundingBox.Height * detectedImage.Height)); detectedImage.Save(detectedImagePath, ImageFormat.Png); } ImageFileUrl = "/images/detectedImage.png"; } else { Result = "判定できませんでした"; } } catch (CustomVisionErrorException e) { Result = "エラー: " + e.Message; } return(Page()); }
private async Task <ImagePrediction> GetPredictionAsync(byte[] imageFile) { var predictionClient = new CustomVisionPredictionClient { ApiKey = _cvSecrets.PredictionKey, Endpoint = _cvSecrets.PredictionUrl }; var stream = new MemoryStream(imageFile); return(await predictionClient.DetectImageAsync(_cvSecrets.ProjectId, _cvSecrets.PublishedName, stream)); }
async Task PredictPhoto(MediaFile photo) { var endpoint = new CustomVisionPredictionClient { ApiKey = await KeyService.GetPredictionKey(), Endpoint = await KeyService.GetEndpoint() }; var results = await endpoint.DetectImageAsync(Guid.Parse(await KeyService.GetProjectId()), await KeyService.GetPublishName(), photo.GetStream()); AllPredictions = results.Predictions .Where(p => p.Probability > Probability) .ToList(); }
static public async void ClassifyImage(Guid guid, string path) { using (var stream = File.Open(path, FileMode.Open)) { var imageresult = await endpoint.DetectImageAsync(guid, publishedModelName, stream); var result = imageresult.Predictions.Select(x => x.Probability).ToList(); var maskresult = result.Max(); if (maskresult >= 0.75) { Console.WriteLine("有戴口罩"); } else { Console.WriteLine("沒戴口罩"); } } }
private async Task <IEnumerable <PredictionModel> > GetMeaningfulpredictions(byte[] image) { using (var stream = new MemoryStream(image)) { var client = new CustomVisionPredictionClient() { ApiKey = _config.PredictionKey, Endpoint = _config.PredictionEndpoint }; var predictionResult = await client.DetectImageAsync(new Guid(_config.ProjectId), _config.ModelName, stream); foreach (var c in predictionResult.Predictions) { _log.LogDebug($"\t{c.TagName}: {c.Probability:P1} [ {c.BoundingBox.Left}, {c.BoundingBox.Top}, {c.BoundingBox.Width}, {c.BoundingBox.Height} ]"); } var meaningfulPredictions = predictionResult.Predictions.Where(x => x.Probability > 0.15); return(meaningfulPredictions); } }
public async Task <string> DetectAsync(IFormFile file, int lightness) { var predictionSection = _config.GetSection("PredictionApi"); var client = new CustomVisionPredictionClient { ApiKey = predictionSection["key"], Endpoint = predictionSection["endpoint"] }; var result = await client.DetectImageAsync(Guid.Parse(predictionSection["projectid"]), predictionSection["iteration"], file.OpenReadStream()); if (lightness == -1) { return(DrawRectangles(file, result)); } else { return(DrawLightRectangles(file, result, lightness)); } }
public static async System.Threading.Tasks.Task Run( [BlobTrigger("pending/{name}")] Stream image, [Queue(Constants.QUEUE_NAME)] IAsyncCollector <string> applicationQueue, string name, ILogger log, ExecutionContext executionContext) { var sourceStream = new MemoryStream(); await image.CopyToAsync(sourceStream); var bitmap = new Bitmap(sourceStream); var customVisionPredictionClient = new CustomVisionPredictionClient { ApiKey = Environment.GetEnvironmentVariable("CustomVisionPredictionClient_ApiKey"), Endpoint = Environment.GetEnvironmentVariable("CustomVisionPredictionClient_Endpoint") }; sourceStream.Position = 0; var response = await customVisionPredictionClient.DetectImageAsync(Guid.Parse(Environment.GetEnvironmentVariable("CustomVisionPredictionClient_ProjectId")), "Completed Route", sourceStream); var routes = new List <string>(); foreach (var predictionModel in response.Predictions) { if (predictionModel.TagName == "Completed Route" && predictionModel.Probability > 0.85) { var cropped = CropBitmap(bitmap, predictionModel.BoundingBox.Left, predictionModel.BoundingBox.Top, predictionModel.BoundingBox.Width, predictionModel.BoundingBox.Height); var memoryStream = new MemoryStream(); //ONLY FOR DEBUG //cropped.Save(Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Desktop),Guid.NewGuid().ToString())); cropped.Save(memoryStream, System.Drawing.Imaging.ImageFormat.Jpeg); memoryStream.Position = 0; //https://stackoverflow.com/questions/53367132/where-to-store-files-for-azure-function var path = Path.Combine(executionContext.FunctionAppDirectory, "Zwift-5c2367dfe003.json"); Environment.SetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS", path); Image tmpImage = await Image.FromStreamAsync(memoryStream); var client = await ImageAnnotatorClient.CreateAsync(); var tmp = await client.DetectTextAsync(tmpImage); var annotation = tmp.FirstOrDefault(); if (annotation?.Description != null) { routes.Add(annotation.Description.Replace("\n", " ").Trim()); } } } if (routes.Count > 0) { var user = name.Split("_").First(); await applicationQueue.AddAsync(JsonConvert.SerializeObject(new MultipleRoutesCompletedModel { UserId = user, Routes = routes })); await applicationQueue.FlushAsync(); } }
public static async Task Main(string[] args) { // Project name must match the one in Custom Vision var projectName = "Open Images Detector"; var publishedModelName = "OpenImagesDetectorModel"; // Replace with your own from Custom Vision project var customVisionKey = ""; var endpoint = ""; var resourceId = ""; if (string.IsNullOrWhiteSpace(endpoint) || string.IsNullOrWhiteSpace(customVisionKey) || string.IsNullOrWhiteSpace(resourceId)) { Console.WriteLine("You need to set the endpoint, key and resource id. The program will end;"); Console.ReadKey(); return; } // Training Client var trainingApi = new CustomVisionTrainingClient() { ApiKey = customVisionKey, Endpoint = endpoint }; Console.WriteLine($"----- Selecting existing project: {projectName}... -----"); var projects = await trainingApi.GetProjectsAsync(); var project = projects.FirstOrDefault(x => x.Name == projectName); if (project != null) { Console.WriteLine($"\t{projectName} found in Custom Vision workspace."); var domainId = project.Settings.DomainId; var projectDomain = await trainingApi.GetDomainAsync(domainId); Console.WriteLine($"\tProject domain: {projectDomain.Name}."); Console.WriteLine($"\tProject type: {projectDomain.Type}."); WriteSeparator(); } else { Console.WriteLine($"\tProject {projectName} was not found in your subscription. The program will end."); return; } Console.WriteLine("Press any key to continue..."); Console.ReadKey(); // Retrieve the tags that already exist in the project Console.WriteLine("----- Retrieving tags... -----"); var modelTags = await trainingApi.GetTagsAsync(project.Id); var tags = new List <Tag>(); var onlineImages = 0; // Obtain tags from our dataset var tagsFile = Path.Combine(trainingFolder, "tags.txt"); var imageLabels = await File.ReadAllLinesAsync(tagsFile); foreach (var label in imageLabels) { // Check if the label already exists var tag = modelTags.FirstOrDefault(x => x.Name == label); if (tag == null) { // If not, create it tag = await trainingApi.CreateTagAsync(project.Id, label); Console.WriteLine($"\tTag {tag.Name} was created."); } else { // If so, just count images with this tag onlineImages += tag.ImageCount; Console.WriteLine($"\tTag {label} was NOT created (it already exists)"); } tags.Add(tag); } WriteSeparator(); Console.WriteLine("Press any key to continue..."); Console.ReadKey(); // Upload images var uploadImages = true; if (onlineImages > 0) { Console.WriteLine($"There are {onlineImages} training images already uploaded. Do you want to upload more? (Y/N)"); uploadImages = Console.ReadKey().Key == ConsoleKey.Y; } Iteration iteration = null; if (uploadImages) { Console.WriteLine("----- Accessing images -----"); var imageFileEntries = new List <ImageFileCreateEntry>(); foreach (var label in imageLabels) { var tagFolder = Path.Combine(trainingFolder, label); var tagImages = Directory.GetFiles(tagFolder); var tagLabels = Path.Combine(tagFolder, "normalizedLabel"); foreach (var image in tagImages) { var imageName = Path.GetFileNameWithoutExtension(image); var imageFile = Path.GetFileName(image); var imageLabelFile = $"{imageName}.txt"; var tagCV = tags.Single(x => x.Name == label); var imageBoundingBoxes = await File.ReadAllLinesAsync(Path.Combine(tagLabels, imageLabelFile)); var imageRegions = new List <Region>(); foreach (var bbox in imageBoundingBoxes) { var normalizedData = bbox.Split(); var left = float.Parse(normalizedData[0]); var top = float.Parse(normalizedData[1]); var width = float.Parse(normalizedData[2]); var height = float.Parse(normalizedData[3]); imageRegions.Add(new Region(tagCV.Id, left, top, width, height)); } Console.WriteLine($"\tAdding image {imageName} with its regions."); imageFileEntries.Add(new ImageFileCreateEntry( imageFile, await File.ReadAllBytesAsync(image), null, imageRegions)); } } var batchPageSize = 64; var numBatches = imageFileEntries.Count / batchPageSize; var sizeLastBatch = imageFileEntries.Count % batchPageSize; for (int batch = 0; batch < numBatches; batch++) { Console.WriteLine($"\tUploading images batch #{batch}."); var entries = imageFileEntries.Skip(batch * batchPageSize).Take(batchPageSize).ToList(); await trainingApi.CreateImagesFromFilesAsync( project.Id, new ImageFileCreateBatch(entries)); } if (sizeLastBatch > 0) { Console.WriteLine($"\tUploading last batch."); var lastEntries = imageFileEntries.Skip(numBatches * batchPageSize).Take(sizeLastBatch).ToList(); await trainingApi.CreateImagesFromFilesAsync(project.Id, new ImageFileCreateBatch(lastEntries)); } WriteSeparator(); Console.WriteLine("Press any key to continue..."); Console.ReadKey(); try { // Now there are images with tags start training the project Console.WriteLine("----- Starting the Training process... -----"); iteration = await trainingApi.TrainProjectAsync(project.Id); // The returned iteration will be in progress, and can be queried periodically to see when it has completed while (iteration.Status == "Training") { Thread.Sleep(1000); Console.WriteLine($"\tIteration '{iteration.Name}' status: {iteration.Status}"); // Re-query the iteration to get it's updated status iteration = await trainingApi.GetIterationAsync(project.Id, iteration.Id); } Console.WriteLine($"\tIteration '{iteration.Name}' status: {iteration.Status}"); WriteSeparator(); Console.WriteLine("Press any key to continue..."); Console.ReadKey(); // The iteration is now trained. Publish it to the prediction endpoint. Console.WriteLine($"----- Starting the Publication process. -----"); await trainingApi.PublishIterationAsync( project.Id, iteration.Id, publishedModelName, resourceId); Console.WriteLine($"\tIteration '{iteration.Name}' published."); WriteSeparator(); Console.WriteLine("Press any key to continue..."); Console.ReadKey(); } catch (Exception ex) { Console.WriteLine($"There was an exception (perhaps nothing changed since last iteration?)."); } } if (iteration == null) { var iterations = await trainingApi.GetIterationsAsync(project.Id); iteration = iterations.OrderByDescending(x => x.LastModified).FirstOrDefault(); Console.WriteLine($"Iteration '{iteration.Name}' found and loaded."); WriteSeparator(); Console.WriteLine("Press any key to continue..."); Console.ReadKey(); } // Prediction Client var predictionClient = new CustomVisionPredictionClient() { ApiKey = customVisionKey, Endpoint = endpoint }; // Make predictions against the new project Console.WriteLine("----- Making predictions -----"); var testImages = LoadImagesFromDisk("Test"); foreach (var image in testImages) { var imageName = Path.GetFileName(image); using (var stream = new MemoryStream(File.ReadAllBytes(image))) { Console.WriteLine($"\tImage: {imageName}"); var result = await predictionClient.DetectImageAsync( project.Id, publishedModelName, stream); // Loop over each prediction and write out the results foreach (var prediction in result.Predictions.OrderByDescending(x => x.Probability)) { Console.WriteLine($"\t\tFor Tag '{prediction.TagName}': {prediction.Probability:P3} " + $"[{prediction.BoundingBox.Left}, {prediction.BoundingBox.Top}, " + $"{prediction.BoundingBox.Width}, {prediction.BoundingBox.Height}]"); } WriteSeparator(); Console.WriteLine("Press any key for next image..."); Console.ReadKey(); } } WriteSeparator(); Console.WriteLine("----- Do you want to export the model? (Y/N) -----"); var exportModel = Console.ReadKey().Key == ConsoleKey.Y; if (exportModel) { do { var platform = string.Empty; var extension = string.Empty; Export export; Console.WriteLine("\tOptions: \n\t1) TensorFlow \n\t2) CoreML \n\t3) Other platform \n\tE) End program"); var option = Console.ReadKey().Key; switch (option) { case ConsoleKey.D1: platform = "TensorFlow"; extension = "zip"; break; case ConsoleKey.D2: platform = "CoreML"; extension = "mlmodel"; break; case ConsoleKey.D3: Console.WriteLine("\tType the platform name"); platform = Console.ReadLine(); Console.WriteLine($"\tNow type the file extension for the {platform} exported model."); extension = Console.ReadLine(); break; case ConsoleKey.E: exportModel = false; break; default: Console.WriteLine("\n\tOption not supported."); break; } WriteSeparator(); if (!string.IsNullOrWhiteSpace(platform)) { try { Console.WriteLine($"\tExporting to {platform}..."); do { var exports = await trainingApi.GetExportsAsync(project.Id, iteration.Id); export = exports.FirstOrDefault(x => x.Platform == platform); if (export == null) { export = await trainingApi.ExportIterationAsync(project.Id, iteration.Id, platform); } Thread.Sleep(1000); Console.WriteLine($"\tStatus: {export.Status}"); } while (export.Status == "Exporting"); Console.WriteLine($"Status: {export.Status}"); if (export.Status == ExportStatus.Done) { Console.WriteLine($"\tDownloading {platform} model"); var filePath = Path.Combine(Environment.CurrentDirectory, $"{publishedModelName}_{platform}.{extension}"); using (var httpClient = new HttpClient()) { using (var stream = await httpClient.GetStreamAsync(export.DownloadUri)) { using (var file = new FileStream(filePath, FileMode.Create)) { await stream.CopyToAsync(file); Console.WriteLine($"\tModel successfully exported. You can find it here:\n\t{filePath}."); WriteSeparator(); } } } } } catch (Exception ex) { Console.WriteLine($"Exception found: {ex.Message}"); WriteSeparator(); } } } while (exportModel); } Console.WriteLine("Press a key to exit the program!"); Console.ReadKey(); }
public static async Task Predict() { // Use test images in this repo - substitute other as needed. Note, the API also takes URLs to pictures, in this case I am sending picture as request payload. List <string> filePaths = Directory.GetFiles(@"..\..\..\..\..\Images\test\", "*.*", SearchOption.TopDirectoryOnly).ToList(); StringBuilder sb = new StringBuilder(); using (CustomVisionPredictionClient endpoint = new CustomVisionPredictionClient() { ApiKey = _predictionKey, Endpoint = _apiEndpoint }) { foreach (string filePath in filePaths) { sb.AppendLine(Path.GetFileName(filePath)); sb.AppendLine(DateTime.UtcNow.ToString()); using (var stream = File.OpenRead(filePath)) { ImagePrediction classificationResult = await endpoint.ClassifyImageAsync(_projectGuid_Classification_Multiclass, _modelName_Classification_Multiclass, stream); // Loop over each classification and write out the results foreach (var c in classificationResult.Predictions.OrderByDescending(p => p.Probability)) { sb.AppendLine($"Classification (multi-class): {c.TagName}: {c.Probability:P1}"); } } sb.AppendLine("--------------------"); using (var stream = File.OpenRead(filePath)) { ImagePrediction classificationResult = await endpoint.ClassifyImageAsync(_projectGuid_Classification_Multilabel, _modelName_Classification_Multilabel, stream); // Loop over each classification and write out the results foreach (var c in classificationResult.Predictions.OrderByDescending(p => p.Probability)) { sb.AppendLine($"Classification (multi-label): {c.TagName}: {c.Probability:P1}"); } } sb.AppendLine("--------------------"); using (var stream = File.OpenRead(filePath)) { ImagePrediction detectionResult = await endpoint.DetectImageAsync(_projectGuid_Detection, _modelName_Detection, stream); // Loop over each detection and write out the results foreach (var c in detectionResult.Predictions.OrderByDescending(p => p.Probability)) { sb.AppendLine($"Detection: {c.TagName}: {c.Probability:P1} " + $"[ {c.BoundingBox.Left}, {c.BoundingBox.Top}, {c.BoundingBox.Width}, {c.BoundingBox.Height} ]"); } } sb.AppendLine("--------------------"); sb.AppendLine(); } } string final = sb.ToString(); File.WriteAllText(_resultsFilePath, final); }