// This part communicates with the customvision.ai REST services async void Button_Azure_Clicked(object sender, EventArgs e) { var imageStream = await PickImage(); if (imageStream == null) { return; } var client = new CustomVisionPredictionClient { ApiKey = KeysAndUrls.CustomVisionPredictionApiKey, Endpoint = KeysAndUrls.PredictionUrl }; var result = await client.ClassifyImageAsync(KeysAndUrls.ProjectId, KeysAndUrls.IterationName, imageStream); var bestResult = result.Predictions.OrderByDescending(p => p.Probability).FirstOrDefault(); if (bestResult == null) { return; } SetLabel($"{bestResult.TagName} ({Math.Round(bestResult.Probability * 100, 2)}%)"); }
public async Task <IDictionary <string, float> > GetPredictionAsync(Stream stream, Guid projectId, Guid iterationId) { if (stream == null) { throw new ArgumentNullException($"{nameof(stream)} cannot be null"); } if (projectId == Guid.Empty || iterationId == Guid.Empty) { throw new ArgumentException($"{nameof(projectId)} and/or {nameof(iterationId)} cannot be empty"); } var predictions = new Dictionary <string, float>(); try { var result = await _endpoint.ClassifyImageAsync(projectId, iterationId.ToString(), stream); var tags = (from p in result.Predictions select new { p.TagName, p.Probability }).OrderByDescending(p => p.Probability); foreach (var t in tags) { predictions.Add(t.TagName, (float)t.Probability); } return(predictions); } catch (Exception ex) { return(predictions); } }
public async Task <string> Analyze(byte[] image, string projectId) { var imageStream = new MemoryStream(image); var prediction = await _client.ClassifyImageAsync(new Guid(projectId), "", imageStream); return(prediction.Predictions.FirstOrDefault().TagName ?? "None"); }
public static async Task <ImagePrediction> PredictImageFile(Guid projectID, string modelName, string file) { var img = new MemoryStream(File.ReadAllBytes(file)); ImagePrediction result = null; try { result = await endpoint.ClassifyImageAsync(projectID, modelName, img); Console.WriteLine($"\nSuccessfully retrieved predictions for image '{file}'."); } catch (Exception e) { Console.WriteLine($"\n{e.GetType().Name}: {e.Message} \nCould not get prediction for image '{file}'."); } // Loop over each prediction and write out the results if (result != null) { foreach (var c in result.Predictions) { Console.WriteLine($"\t{c.TagName}: {c.Probability:P1}"); } } return(result); }
public static async Task <string> DetectShapeAsync(SKImage drawing) { await InitializeAsync(); // we can't detect any shapes if (drawing == null || predictionApi == null || modelName == null) { return(null); } // detect the shape using (var data = drawing.Encode(SKEncodedImageFormat.Png, 100)) using (var stream = data.AsStream()) { var classification = await predictionApi.ClassifyImageAsync(project.Id, modelName, stream); var predictions = classification.Predictions.OrderBy(p => p.Probability); var best = predictions.LastOrDefault(); if (best != null) { return(best.TagName); } } return(null); }
async Task CaptureAndSendScreen(MediaFile file) { var endpoint = new CustomVisionPredictionClient { ApiKey = KeyService.PredictionKey, Endpoint = KeyService.Endpoint }; Console.WriteLine(endpoint.ApiKey); var result = await endpoint.ClassifyImageAsync(Guid.Parse(KeyService.ProjectId), KeyService.PublishName, file.GetStream()); List <PredictionModel> res = result.Predictions.Where(p => p.Probability > .9).ToList(); foreach (PredictionModel model in res) { if (model.Probability > .9) { Console.WriteLine("RECYCLABLE!"); //App.user = new User(App.user.User_Name, App.user.User_Password, App.user.User_Email, App.user.User_PhoneNumber, App.user.User_CoinCount+2, App.user.User_Num_ChallengeCompleted); Console.WriteLine(App.user); SQLManager.updateCoins(App.user, 2, App.connectionInfo); } else { Console.WriteLine("NOT RECYCLABLE!"); } } }
private static async Task <Microsoft.Azure.CognitiveServices.Vision.CustomVision.Prediction.Models.ImagePrediction> PredictImageAsync(string imagePath) { if (!File.Exists(imagePath)) { Console.WriteLine("Image not exits, invalid image path"); return(null); } var iteration = await GetLastPublishedIterationAsync(); Stream testImage = File.OpenRead(imagePath); return(await predictionClient.ClassifyImageAsync(project.Id, iteration.PublishName, testImage)); }
PredictImage(MemoryStream memoryStream) { CustomVisionPredictionClient endpoint = new CustomVisionPredictionClient() { ApiKey = _apiKey, Endpoint = _apiEndpoint }; var result = await endpoint.ClassifyImageAsync(_projectId, _modelName, memoryStream); // Loop over each prediction and write out the results foreach (var c in result.Predictions) { Console.WriteLine($"\t{c.TagName}: {c.Probability:P1}"); } return(result); }
public async Task <bool> IsDuckFace(MediaFile photo) { InitIfRequired(); if (photo == null) { return(false); } using (var stream = photo.GetStreamWithImageRotatedForExternalStorage()) { var predictionModels = await customVisionPredictionClient.ClassifyImageAsync(ApiKeys.ProjectId, "Iteration1", stream); return(predictionModels.Predictions .FirstOrDefault(p => p.TagName == "duckface") .Probability > ProbabilityThreshold); } }
static async Task <ImagePrediction> GetImagePredictionsAsync(string imageFile) { // Create a prediction client var predictionClient = new CustomVisionPredictionClient() { Endpoint = Endpoint, ApiKey = PredictionKey }; // Get predictions from the image using (var imageStream = new FileStream(imageFile, FileMode.Open)) { var predictions = await predictionClient.ClassifyImageAsync (new Guid(ProjetId), PublishedName, imageStream); return(predictions); }; }
private async Task <string> RecognizePicture(MediaFile file) { var message = "Nothing recognized..."; try { if (file != null) { IEnumerable <PredictionModel> tags = null; using (var stream = file.GetStream()) { tags = (await _endpoint.ClassifyImageAsync(App.ProjectGuid, App.PublishedName, stream)) .Predictions .OrderByDescending(p => p.Probability); } var bestTag = tags .FirstOrDefault(p => p.Probability > 0.5); if (bestTag != null) { message = $"{bestTag.TagName} ({bestTag.Probability:P1})"; } else { message = $"Not quite sure..."; foreach (var tag in tags) { message += Environment.NewLine + $"({tag.TagName}, {tag.Probability:P1})"; } } } } catch (Exception ex) { message = ex.Message; } return(message); }
public async Task IdentifyFood(Stream image) { string predictionKey = "4a6f15ad57c941b1b72e215f6f69c410"; string endpoint = "https://southeastasia.api.cognitive.microsoft.com/"; Guid projectId = new Guid("5add8d84-ecc6-4782-bd74-5ffbf24abf8a"); string modelName = "Published"; CustomVisionPredictionClient client = new CustomVisionPredictionClient() { ApiKey = predictionKey, Endpoint = endpoint }; var result = await client.ClassifyImageAsync(projectId, modelName, image); var firstPrediction = result.Predictions.FirstOrDefault(); resultText.Text = $"{(firstPrediction.Probability * 100).ToString("0.##")}% {firstPrediction.TagName}"; }
//public void OnGet() //{ //} public async Task <IActionResult> OnPostAsync() { // Show image on web page var imageFilePath = Path.Combine(_host.WebRootPath, "images\\uploadedImage.png"); using (var fileStream = new FileStream(imageFilePath, FileMode.OpenOrCreate)) { await ImageFile.CopyToAsync(fileStream); } ImageFileUrl = "/images/uploadedImage.png"; // Post image to Custom Vision, get resut and show on web page var cvClient = new CustomVisionPredictionClient() { Endpoint = cvEndpoint, ApiKey = cvPredictionKey }; try { var cvResult = await cvClient.ClassifyImageAsync( Guid.Parse(cvProjectId), cvPublishName, ImageFile.OpenReadStream()); if (cvResult.Predictions.Count > 0) { Result = "結果:"; Predictions = cvResult.Predictions; } else { Result = "判定できませんでした"; } } catch (CustomVisionErrorException e) { Result = "エラー: " + e.Message; } return(Page()); }
public async Task <string> GetClassifyImage(MediaFile photo) { try { var endpoint = new CustomVisionPredictionClient { ApiKey = CustomVision.PredictionKey, Endpoint = CustomVision.CustomVisionEndPoint }; var results = await endpoint.ClassifyImageAsync(Guid.Parse(CustomVision.ProjectId), CustomVision.PublishName, photo.GetStream()); return(results.Predictions?.OrderByDescending(x => x.Probability).FirstOrDefault().TagName); } catch (Exception ex) { Debug.WriteLine(ex.Message); return(string.Empty); } }
public async Task <Product> IdentifyProduct(Stream image) { CustomVisionPredictionClient endpoint = new CustomVisionPredictionClient() { ApiKey = _predictionKey, Endpoint = _endpointUrl }; image.Position = 0; var result = await endpoint.ClassifyImageAsync(_projectId, PublishedModelName, image); var tag = result.Predictions .Where(x => x.Probability > 0.2) .OrderByDescending(x => x.Probability) .FirstOrDefault()?.TagId; if (tag == null) { return(null); } return(await _productRepository.Get(tag.Value)); }
private async void OnSelectImageButtonClicked(object sender, RoutedEventArgs e) { var picker = new FileOpenPicker(); picker.ViewMode = PickerViewMode.Thumbnail; picker.SuggestedStartLocation = PickerLocationId.PicturesLibrary; picker.FileTypeFilter.Add(".jpg"); picker.FileTypeFilter.Add(".jpeg"); picker.FileTypeFilter.Add(".png"); var file = await picker.PickSingleFileAsync(); if (file != null) { using (IRandomAccessStream stream = await file.OpenAsync(FileAccessMode.Read)) { // Show the image var image = new BitmapImage(); await image.SetSourceAsync(stream); LoadedImage.Source = image; stream.Seek(0L); try { Progress.IsActive = true; Overlay.Visibility = Visibility.Visible; // Submit the image to the Custom Vision Service CustomVisionPredictionClient client = new CustomVisionPredictionClient() { ApiKey = _key, Endpoint = _uri }; var result = await client.ClassifyImageAsync(_id, _name, stream.AsStream()); Progress.IsActive = false; Overlay.Visibility = Visibility.Collapsed; // Show the result var probability = result.Predictions.FirstOrDefault(x => x.TagName.ToLowerInvariant() == "hotdog").Probability; if (probability > 0.90) { await new MessageDialog("It's a hot dog!").ShowAsync(); } else { await new MessageDialog("Not a hot dog").ShowAsync(); } } catch (Exception ex) { Progress.IsActive = false; Overlay.Visibility = Visibility.Collapsed; await new MessageDialog(ex.Message).ShowAsync(); } finally { Progress.IsActive = false; Overlay.Visibility = Visibility.Collapsed; } } } }
private async Task <DialogTurnResult> step2Async(WaterfallStepContext sc, CancellationToken cancellationToken) { var attachmentList = (sc.Result as IList <Attachment>); async Task <byte[]> readFileAsync() { using (var client = new HttpClient()) { using (var response = await client.GetAsync(attachmentList[0].ContentUrl)) { if (response.IsSuccessStatusCode) { return(await response.Content.ReadAsByteArrayAsync()); } } } return(null); } var result = default(ImagePrediction); var imageBytes = await readFileAsync(); using (var imageData = new MemoryStream(imageBytes)) { var predictionClient = new CustomVisionPredictionClient() { ApiKey = CustomVisionOptions.PredictionKey, Endpoint = CustomVisionOptions.RegionEndpoint }; result = await predictionClient.ClassifyImageAsync(new Guid(CustomVisionOptions.ProjectId), CustomVisionOptions.PublishedName, imageData); } var tagName = String.Empty; foreach (var item in result.Predictions) { if (item.Probability < CustomVisionOptions.ProbabilityThreshold) { continue; } tagName = item.TagName; break; } if (String.IsNullOrEmpty(tagName)) { sc.Values[KEY_IMAGE_BYTES] = imageBytes; return(await sc.PromptAsync( nameof(TextPrompt), new PromptOptions { Prompt = MessageFactory.Text($"I'm not sure what it is. Could you tell me the car model?"), })); } else { await sc.Context.SendActivityAsync($"Wow! Nice {tagName}!"); return(await sc.ReplaceDialogAsync(nameof(CarRecognitionDialog), new DialogOptions { IsInLoop = true })); } }
private async void OnSelectImageButtonClicked(object sender, RoutedEventArgs e) { var picker = new FileOpenPicker(); picker.ViewMode = PickerViewMode.Thumbnail; picker.SuggestedStartLocation = PickerLocationId.PicturesLibrary; picker.FileTypeFilter.Add(".jpg"); picker.FileTypeFilter.Add(".jpeg"); picker.FileTypeFilter.Add(".png"); var file = await picker.PickSingleFileAsync(); if (file != null) { using (IRandomAccessStream stream = await file.OpenAsync(FileAccessMode.Read)) { // Show the image var image = new BitmapImage(); await image.SetSourceAsync(stream); LoadedImage.Source = image; stream.Seek(0L); try { Progress.IsActive = true; Overlay.Visibility = Visibility.Visible; // Submit the image to the Custom Vision Service CustomVisionPredictionClient client = new CustomVisionPredictionClient( new ApiKeyServiceClientCredentials(_key), new System.Net.Http.DelegatingHandler[] { } ); client.Endpoint = _uri; var result = await client.ClassifyImageAsync(_id, _name, stream.AsStream()); var prediction = result.Predictions.FirstOrDefault(x => x.TagName.ToLowerInvariant() == "hotdog"); Progress.IsActive = false; Overlay.Visibility = Visibility.Collapsed; // Show the results if (prediction != null) { // If the results include a "hotdog" label, show the probability that it's a hot dog await new MessageDialog($"Probability that it's a hot dog: {prediction.Probability:P1}").ShowAsync(); } else { // If the results don't include a "hotdog" label, show all tags and probabilities var builder = new StringBuilder(); foreach (var pred in result.Predictions) { builder.Append($"{pred.TagName}: {pred.Probability:P1}\n"); } await new MessageDialog(builder.ToString()).ShowAsync(); } } catch (Exception ex) { Progress.IsActive = false; Overlay.Visibility = Visibility.Collapsed; await new MessageDialog(ex.Message).ShowAsync(); } finally { Progress.IsActive = false; Overlay.Visibility = Visibility.Collapsed; } } } }
public static async Task Main() { // These are the image tags var labels = new Dictionary <int, string>() { { 1, "dristan cold box" }, { 2, "onion" }, { 4, "tomato" }, { 5, "rolaids bottle" }, { 7, "arizona iced tea" }, { 10, "cup" }, { 14, "cat" }, { 19, "firetruck car" }, { 28, "frog" }, { 31, "tylenol" }, { 33, "glue" }, { 35, "porcelain plate" }, { 37, "toy tank" }, { 46, "marlboro cigarette box" }, { 47, "donut toy" }, { 48, "piggy bank" }, { 49, "canada dry ginger ale" } }; // Project name must match the one in Custom Vision var projectName = "COIL100 Small"; // Replace with your own from Custom Vision project var endpoint = ""; var customVisionKey = ""; var resourceId = ""; var publishedModelName = "coil100Model"; if (string.IsNullOrWhiteSpace(endpoint) || string.IsNullOrWhiteSpace(customVisionKey) || string.IsNullOrWhiteSpace(resourceId)) { Console.WriteLine("You need to set the endpoint, key and resource id. The program will end;"); Console.ReadKey(); return; } // Training Client var trainingApi = new CustomVisionTrainingClient() { ApiKey = customVisionKey, Endpoint = endpoint }; Console.WriteLine($"Selecting existing project: {projectName}..."); var projects = await trainingApi.GetProjectsAsync(); var project = projects.FirstOrDefault(x => x.Name == projectName); if (project != null) { Console.WriteLine($"{projectName} found in Custom Vision workspace."); WriteSeparator(); } else { Console.WriteLine($"Project {projectName} was not found in your subscription. The program will end."); return; } Console.WriteLine("Retrieving tags..."); // Retrieve the tags that already exist in the project var modelTags = await trainingApi.GetTagsAsync(project.Id); var tags = new List <Tag>(); var onlineImages = 0; foreach (var label in labels) { // Check if the label already exists var tag = modelTags.FirstOrDefault(x => x.Name == label.Value); if (tag == null) { // If not, create it tag = await trainingApi.CreateTagAsync(project.Id, label.Value); Console.WriteLine($"Tag {tag.Name} was created."); } else { // If so, just count images with this tag onlineImages += tag.ImageCount; Console.WriteLine($"Tag {label.Value} was NOT created (it already exists)"); } tags.Add(tag); } WriteSeparator(); var uploadImages = true; if (onlineImages > 0) { Console.WriteLine($"There are {onlineImages} training images already uploaded. Do you want to upload more? (Y/N)"); uploadImages = Console.ReadKey().Key == ConsoleKey.Y; } Iteration iteration = null; if (uploadImages) { Console.WriteLine("\tUploading images"); var images = LoadImagesFromDisk("images"); foreach (var image in images) { var fileName = Path.GetFileName(image); var objImage = fileName.Split(new string[] { "__" }, StringSplitOptions.None); var id = int.Parse(objImage[0].Remove(0, 3)); var label = labels.SingleOrDefault(t => t.Key == id); var tag = tags.Single(x => x.Name == label.Value); using (var stream = new MemoryStream(File.ReadAllBytes(image))) { await trainingApi.CreateImagesFromDataAsync( project.Id, stream, new List <Guid>() { tag.Id }); Console.WriteLine($"Image {fileName} uploaded"); } } WriteSeparator(); try { // Now there are images with tags start training the project Console.WriteLine("\tTraining started..."); iteration = await trainingApi.TrainProjectAsync(project.Id); // The returned iteration will be in progress, and can be queried periodically to see when it has completed while (iteration.Status == "Training") { Thread.Sleep(1000); Console.WriteLine($"Iteration '{iteration.Name}' status: {iteration.Status}"); // Re-query the iteration to get it's updated status iteration = await trainingApi.GetIterationAsync(project.Id, iteration.Id); } Console.WriteLine($"Iteration '{iteration.Name}' status: {iteration.Status}"); WriteSeparator(); // The iteration is now trained. Publish it to the prediction endpoint. await trainingApi.PublishIterationAsync( project.Id, iteration.Id, publishedModelName, resourceId); Console.WriteLine($"Iteration '{iteration.Name}' published."); WriteSeparator(); } catch (Exception ex) { Console.WriteLine($"There was an exception (perhaps nothing changed since last iteration?)."); } } if (iteration == null) { var iterations = await trainingApi.GetIterationsAsync(project.Id); iteration = iterations.LastOrDefault(); Console.WriteLine($"Iteration '{iteration.Name}' found and loaded."); WriteSeparator(); } // Prediction Client var predictionClient = new CustomVisionPredictionClient() { ApiKey = customVisionKey, Endpoint = endpoint }; // Make predictions against the new project Console.WriteLine("Making predictions:"); var testImages = LoadImagesFromDisk("Test"); foreach (var image in testImages) { var imageName = Path.GetFileName((image)); using (var stream = new MemoryStream(File.ReadAllBytes(image))) { Console.WriteLine($"\t---------- Image {imageName} ----------"); var result = await predictionClient.ClassifyImageAsync( project.Id, publishedModelName, stream); // Loop over each prediction and write out the results foreach (var c in result.Predictions) { Console.WriteLine($"For Tag {c.TagName}: \t{c.Probability:P3}"); } } } WriteSeparator(); Console.WriteLine("Do you want to export the model? (Y/N)"); var exportModel = Console.ReadKey().Key == ConsoleKey.Y; if (exportModel) { do { var platform = string.Empty; var extension = string.Empty; Export export; Console.WriteLine("Options: \n1) TensorFlow \n2) CoreML \n3) Other platform \nE) End program"); var option = Console.ReadKey().Key; switch (option) { case ConsoleKey.D1: platform = "TensorFlow"; extension = "zip"; break; case ConsoleKey.D2: platform = "CoreML"; extension = "mlmodel"; break; case ConsoleKey.D3: Console.WriteLine("Type the platform name"); platform = Console.ReadLine(); Console.WriteLine($"Now type the file extension for the {platform} exported model."); extension = Console.ReadLine(); break; case ConsoleKey.E: exportModel = false; break; default: Console.WriteLine("\nOption not supported."); break; } WriteSeparator(); if (!string.IsNullOrWhiteSpace(platform)) { try { do { var exports = await trainingApi.GetExportsAsync(project.Id, iteration.Id); export = exports.FirstOrDefault(x => x.Platform == platform); if (export == null) { export = await trainingApi.ExportIterationAsync(project.Id, iteration.Id, platform); } Thread.Sleep(1000); Console.WriteLine($"Status: {export.Status}"); } while (export.Status == "Exporting"); Console.WriteLine($"Status: {export.Status}"); if (export.Status == ExportStatus.Done) { Console.WriteLine($"Downloading {platform} model"); var filePath = Path.Combine(Environment.CurrentDirectory, $"{publishedModelName}_{platform}.{extension}"); using (var httpClient = new HttpClient()) { using (var stream = await httpClient.GetStreamAsync(export.DownloadUri)) { using (var file = new FileStream(filePath, FileMode.Create)) { await stream.CopyToAsync(file); Console.WriteLine($"Model exported successfully. Check {filePath}."); WriteSeparator(); } } } } } catch (Exception ex) { Console.WriteLine($"Exception found: {ex.Message}"); WriteSeparator(); } } } while (exportModel); } Console.WriteLine("Press a key to exit the program!"); Console.ReadKey(); }
public static async Task Predict() { // Use test images in this repo - substitute other as needed. Note, the API also takes URLs to pictures, in this case I am sending picture as request payload. List <string> filePaths = Directory.GetFiles(@"..\..\..\..\..\Images\test\", "*.*", SearchOption.TopDirectoryOnly).ToList(); StringBuilder sb = new StringBuilder(); using (CustomVisionPredictionClient endpoint = new CustomVisionPredictionClient() { ApiKey = _predictionKey, Endpoint = _apiEndpoint }) { foreach (string filePath in filePaths) { sb.AppendLine(Path.GetFileName(filePath)); sb.AppendLine(DateTime.UtcNow.ToString()); using (var stream = File.OpenRead(filePath)) { ImagePrediction classificationResult = await endpoint.ClassifyImageAsync(_projectGuid_Classification_Multiclass, _modelName_Classification_Multiclass, stream); // Loop over each classification and write out the results foreach (var c in classificationResult.Predictions.OrderByDescending(p => p.Probability)) { sb.AppendLine($"Classification (multi-class): {c.TagName}: {c.Probability:P1}"); } } sb.AppendLine("--------------------"); using (var stream = File.OpenRead(filePath)) { ImagePrediction classificationResult = await endpoint.ClassifyImageAsync(_projectGuid_Classification_Multilabel, _modelName_Classification_Multilabel, stream); // Loop over each classification and write out the results foreach (var c in classificationResult.Predictions.OrderByDescending(p => p.Probability)) { sb.AppendLine($"Classification (multi-label): {c.TagName}: {c.Probability:P1}"); } } sb.AppendLine("--------------------"); using (var stream = File.OpenRead(filePath)) { ImagePrediction detectionResult = await endpoint.DetectImageAsync(_projectGuid_Detection, _modelName_Detection, stream); // Loop over each detection and write out the results foreach (var c in detectionResult.Predictions.OrderByDescending(p => p.Probability)) { sb.AppendLine($"Detection: {c.TagName}: {c.Probability:P1} " + $"[ {c.BoundingBox.Left}, {c.BoundingBox.Top}, {c.BoundingBox.Width}, {c.BoundingBox.Height} ]"); } } sb.AppendLine("--------------------"); sb.AppendLine(); } } string final = sb.ToString(); File.WriteAllText(_resultsFilePath, final); }