public async void TagRetrieval() { using (MockContext context = MockContext.Start(this.GetType())) { HttpMockServer.Initialize(this.GetType(), "TagRetrieval", RecorderMode); using (ICustomVisionTrainingClient client = GetTrainingClient()) { var newProject = await client.CreateProjectAsync("TagRetrievalTest", projDescription, ProjectBuilderHelper.GeneralDomain); var numTags = 4; for (var i = 0; i < numTags; i++) { await client.CreateTagAsync(newProject.Id, "Tag #" + i.ToString(), string.Empty); } var tagList = await client.GetTagsAsync(newProject.Id); Assert.Equal(numTags, tagList.Count); foreach (var tag in tagList) { Assert.NotEqual(Guid.Empty, tag.Id); Assert.Contains("Tag #", tag.Name); Assert.Null(tag.Description); Assert.Equal(0, tag.ImageCount); } await client.DeleteProjectAsync(newProject.Id); } } }
public async void CreateImagesFromFiles() { var dataFileName = "hemlock_1.jpg"; using (MockContext context = MockContext.Start(this.GetType().Name)) { HttpMockServer.Initialize(this.GetType().Name, "CreateImagesFromFiles", RecorderMode); ICustomVisionTrainingClient client = GetTrainingClient(); var newProject = client.CreateProjectAsync(projName, projDescription, FoodDomain).Result; var tag = client.CreateTagAsync(newProject.Id, tagName).Result; var fileName = Path.Combine("TestImages", "tag1", dataFileName); var fileImages = new ImageFileCreateEntry[] { new ImageFileCreateEntry(dataFileName, File.ReadAllBytes(fileName)) }; var tags = new Guid[] { tag.Id }; var imageCreatedFromFile = client.CreateImagesFromFilesAsync(newProject.Id, new ImageFileCreateBatch(fileImages, tags)).Result; Assert.True(imageCreatedFromFile.IsBatchSuccessful); Assert.Equal(1, imageCreatedFromFile.Images.Count); Assert.Contains(dataFileName, imageCreatedFromFile.Images[0].SourceUrl); Assert.Equal("OK", imageCreatedFromFile.Images[0].Status); Assert.NotEqual(Guid.Empty, imageCreatedFromFile.Images[0].Image.Id); Assert.NotEqual(0, imageCreatedFromFile.Images[0].Image.Width); Assert.NotEqual(0, imageCreatedFromFile.Images[0].Image.Height); Assert.NotEmpty(imageCreatedFromFile.Images[0].Image.OriginalImageUri); Assert.NotEmpty(imageCreatedFromFile.Images[0].Image.ResizedImageUri); Assert.NotEmpty(imageCreatedFromFile.Images[0].Image.ThumbnailUri); await client.DeleteProjectAsync(newProject.Id); } }
public static async Task <ProjectIteration[]> GetProjectIterations(ICustomVisionTrainingClient trainingClient, string predictionResourceId, Guid[] projects) { var result = new List <ProjectIteration>(); foreach (var project in projects) { var iterations = await trainingClient.GetIterationsAsync(project); var projectEntity = await trainingClient.GetProjectAsync(project); var domain = await trainingClient.GetDomainAsync(projectEntity.Settings.DomainId); var iteration = iterations.Where(i => i.Status == "Completed").OrderByDescending(i => i.TrainedAt.Value).FirstOrDefault(); if (iteration != null) { if (string.IsNullOrEmpty(iteration.PublishName)) { await trainingClient.PublishIterationAsync(project, iteration.Id, publishName : iteration.Id.ToString(), predictionId : predictionResourceId); iteration = await trainingClient.GetIterationAsync(project, iteration.Id); } result.Add(new ProjectIteration() { Project = project, Iteration = iteration.Id, ProjectName = projectEntity.Name, PublishedName = iteration.PublishName, IsObjectDetection = domain.Type == "ObjectDetection" }); } } return(result.ToArray()); }
public async void CreateImagesFromData() { var dataFileName = "hemlock_1.jpg"; using (MockContext context = MockContext.Start(this.GetType())) { HttpMockServer.Initialize(this.GetType(), "CreateImagesFromData", RecorderMode); using (ICustomVisionTrainingClient client = GetTrainingClient()) { var newProject = client.CreateProjectAsync(projName, projDescription, ProjectBuilderHelper.FoodDomain).Result; var tag = client.CreateTagAsync(newProject.Id, tagName).Result; using (FileStream stream = new FileStream(Path.Combine("TestImages", "tag1", dataFileName), FileMode.Open)) { var imageCreatedFromData = client.CreateImagesFromDataAsync(newProject.Id, stream, new Guid[] { tag.Id }).Result; Assert.True(imageCreatedFromData.IsBatchSuccessful); Assert.Equal(1, imageCreatedFromData.Images.Count); Assert.Contains(dataFileName, imageCreatedFromData.Images[0].SourceUrl); Assert.Equal("OK", imageCreatedFromData.Images[0].Status); Assert.NotEqual(Guid.Empty, imageCreatedFromData.Images[0].Image.Id); Assert.NotEqual(0, imageCreatedFromData.Images[0].Image.Width); Assert.NotEqual(0, imageCreatedFromData.Images[0].Image.Height); Assert.NotEmpty(imageCreatedFromData.Images[0].Image.OriginalImageUri); Assert.NotEmpty(imageCreatedFromData.Images[0].Image.ResizedImageUri); Assert.NotEmpty(imageCreatedFromData.Images[0].Image.ThumbnailUri); } await client.DeleteProjectAsync(newProject.Id); } } }
public async void GetImagesByIds() { using (MockContext context = MockContext.Start(this.GetType())) { HttpMockServer.Initialize(this.GetType(), "GetImagesByIds", RecorderMode); using (var project = CreateTrainedImageClassificationProject()) { ICustomVisionTrainingClient client = BaseTests.GetTrainingClient(); var tags = await client.GetTagsAsync(project.ProjectId); var imagesForTag = await client.GetTaggedImagesAsync(project.ProjectId, null, new List <Guid>(new Guid[] { tags[0].Id })); Assert.Equal(5, imagesForTag.Count); var images = await client.GetImagesByIdsAsync(project.ProjectId, imagesForTag.Select(img => img.Id).ToList()); Assert.Equal(5, images.Count); foreach (var image in images) { Assert.Equal(1, image.Tags.Count); var tag = image.Tags[0]; Assert.Equal(tags[0].Id, tag.TagId); Assert.Equal(tags[0].Name, tag.TagName); Assert.NotNull(image.ThumbnailUri); Assert.NotEmpty(image.OriginalImageUri); Assert.NotEmpty(image.ResizedImageUri); Assert.True(image.Width > 0); Assert.True(image.Height > 0); } } } }
public async void CreateUpdateDeleteProject() { var updatedProjName = "Another Name"; var updatedProjDescription = "Updated Project Description"; using (MockContext context = MockContext.Start(this.GetType())) { HttpMockServer.Initialize(this.GetType(), "CreateUpdateDeleteProject", RecorderMode); using (ICustomVisionTrainingClient client = GetTrainingClient()) { var newProject = client.CreateProjectAsync(projName, projDescription).Result; Assert.Contains(projName, newProject.Name); Assert.Equal(projDescription, newProject.Description); Assert.NotEqual(Guid.Empty, newProject.Id); var updatedProject = client.UpdateProjectAsync(newProject.Id, new Project() { Name = updatedProjName, Description = updatedProjDescription, }).Result; Assert.Equal(updatedProjName, updatedProject.Name); Assert.Equal(updatedProjDescription, updatedProject.Description); Assert.Equal(newProject.Id, updatedProject.Id); Assert.Equal(newProject.Settings.DomainId, updatedProject.Settings.DomainId); await client.DeleteProjectAsync(newProject.Id); } } }
public async void ProjectRetrieval() { using (MockContext context = MockContext.Start(this.GetType())) { HttpMockServer.Initialize(this.GetType(), "ProjectRetrieval", RecorderMode); using (ICustomVisionTrainingClient client = GetTrainingClient()) { var newProject = await client.CreateProjectAsync(projName, projDescription, ProjectBuilderHelper.FoodDomain); var projects = await client.GetProjectsAsync(); Assert.True(projects.Count > 0); Assert.Contains(projects, p => p.Id == newProject.Id); var firstProject = await client.GetProjectAsync(projects[0].Id); Assert.Equal(projects[0].Id, firstProject.Id); Assert.Equal(projects[0].Name, firstProject.Name); Assert.Equal(projects[0].Description, firstProject.Description); Assert.Equal(projects[0].Settings.DomainId, firstProject.Settings.DomainId); Assert.Equal(projects[0].Settings.TargetExportPlatforms.Count, firstProject.Settings.TargetExportPlatforms.Count); Assert.Equal(projects[0].DrModeEnabled, firstProject.DrModeEnabled); await client.DeleteProjectAsync(newProject.Id); } } }
public async void GetIterationPerformance() { using (MockContext context = MockContext.Start(this.GetType())) { HttpMockServer.Initialize(this.GetType(), "GetIterationPerformance", RecorderMode); using (var project = CreateTrainedImageClassificationProject()) { ICustomVisionTrainingClient client = BaseTests.GetTrainingClient(); var iterations = await client.GetIterationsAsync(project.ProjectId); Assert.True(iterations.Count > 0); var iterationPerf = await client.GetIterationPerformanceAsync(project.ProjectId, iterations[iterations.Count - 1].Id, 0.9); Assert.Equal(1, iterationPerf.Recall); Assert.Equal(0, iterationPerf.RecallStdDeviation); Assert.Equal(1, iterationPerf.Precision); Assert.Equal(0, iterationPerf.PrecisionStdDeviation); Assert.Equal(2, iterationPerf.PerTagPerformance.Count); Assert.Equal("Tag1", iterationPerf.PerTagPerformance[0].Name); Assert.Equal(1, iterationPerf.PerTagPerformance[0].Recall); Assert.Equal(0, iterationPerf.PerTagPerformance[0].RecallStdDeviation); Assert.Equal(1, iterationPerf.PerTagPerformance[0].Precision); Assert.Equal(0, iterationPerf.PerTagPerformance[0].PrecisionStdDeviation); Assert.Equal("Tag2", iterationPerf.PerTagPerformance[1].Name); Assert.Equal(1, iterationPerf.PerTagPerformance[1].Recall); Assert.Equal(0, iterationPerf.PerTagPerformance[1].RecallStdDeviation); Assert.Equal(1, iterationPerf.PerTagPerformance[1].Precision); Assert.Equal(0, iterationPerf.PerTagPerformance[1].PrecisionStdDeviation); } } }
public async void QuickTests() { var dataFileName = "test_image.jpg"; using (MockContext context = MockContext.Start(this.GetType())) { HttpMockServer.Initialize(this.GetType(), "QuickTests", RecorderMode); using (var project = CreateTrainedImageClassificationProject()) using (ICustomVisionTrainingClient client = BaseTests.GetTrainingClient()) { using (FileStream stream = new FileStream(Path.Combine("TestImages", dataFileName), FileMode.Open)) { var imageResult = await client.QuickTestImageAsync(project.ProjectId, stream, project.IterationId); Assert.NotEqual(Guid.Empty, imageResult.Id); Assert.NotEqual(Guid.Empty, imageResult.Iteration); Assert.Equal(project.ProjectId, imageResult.Project); Assert.NotEqual(0, imageResult.Predictions.Count); Assert.InRange(imageResult.Predictions[0].Probability, 0.9, 1); Assert.Equal("Tag1", imageResult.Predictions[0].TagName); } string imageUrl = "https://raw.githubusercontent.com/Microsoft/Cognitive-CustomVision-Windows/master/Samples/Images/Test/test_image.jpg"; var urlResult = await client.QuickTestImageUrlAsync(project.ProjectId, new ImageUrl(imageUrl), project.IterationId); Assert.NotEqual(Guid.Empty, urlResult.Id); Assert.NotEqual(Guid.Empty, urlResult.Iteration); Assert.Equal(project.ProjectId, urlResult.Project); Assert.NotEqual(0, urlResult.Predictions.Count); Assert.InRange(urlResult.Predictions[0].Probability, 0.9, 1); Assert.Equal("Tag1", urlResult.Predictions[0].TagName); } } }
public async void QueryPredictionResults() { using (MockContext context = MockContext.Start(this.GetType())) { HttpMockServer.Initialize(this.GetType(), "QueryPredictionResults", RecorderMode); using (var project = CreateTrainedImageClassificationProject()) { ICustomVisionTrainingClient client = BaseTests.GetTrainingClient(); var token = new PredictionQueryToken() { OrderBy = "Newest", }; var result = await client.QueryPredictionsAsync(project.ProjectId, token); Assert.NotEqual(0, result.Results.Count); foreach (var prediction in result.Results) { Assert.Equal(project.ProjectId, prediction.Project); Assert.NotEqual(Guid.Empty, prediction.Id); Assert.NotEqual(Guid.Empty, prediction.Iteration); Assert.NotEmpty(prediction.ThumbnailUri); Assert.NotEmpty(prediction.OriginalImageUri); Assert.NotEmpty(prediction.ResizedImageUri); Assert.NotEqual(0, prediction.Predictions.Count); } } } }
public async void ExportTests() { using (MockContext context = MockContext.Start(this.GetType())) { HttpMockServer.Initialize(this.GetType(), "ExportTests", RecorderMode); using (var project = CreateTrainedImageClassificationProject(ProjectBuilderHelper.ExportableDomain)) { ICustomVisionTrainingClient client = BaseTests.GetTrainingClient(); var iterations = await client.GetIterationsAsync(project.ProjectId); var exportableIteration = iterations.FirstOrDefault(e => e.Exportable); Assert.NotNull(exportableIteration); var export = await client.ExportIterationAsync(project.ProjectId, exportableIteration.Id, ExportPlatform.TensorFlow); Assert.Equal("Exporting", export.Status); Assert.Null(export.DownloadUri); Assert.Equal(ExportPlatform.TensorFlow, export.Platform); Assert.False(export.NewerVersionAvailable); while (export.Status != "Done") { var exports = await client.GetExportsAsync(project.ProjectId, exportableIteration.Id); Assert.Equal(1, exports.Count); export = exports.Where(e => e.Platform == ExportPlatform.TensorFlow).FirstOrDefault(); Assert.NotNull(export); Thread.Sleep(1000); } Assert.NotEmpty(export.DownloadUri); } } }
public async void ImageTagManipulation() { using (MockContext context = MockContext.Start(this.GetType())) { HttpMockServer.Initialize(this.GetType(), "ImageTagManipulation", RecorderMode); using (var project = CreateTrainedImageClassificationProject()) { ICustomVisionTrainingClient client = BaseTests.GetTrainingClient(); var untaggedImages = client.GetUntaggedImagesAsync(project.ProjectId).Result; Assert.Equal(0, untaggedImages.Count); var taggedImages = client.GetTaggedImagesAsync(project.ProjectId).Result; Assert.Equal(10, taggedImages.Count); var imageToBeModified = taggedImages[0].Id; var tagToBeModified = taggedImages[0].Tags[0].TagId; await client.DeleteImageTagsAsync(project.ProjectId, new Guid[] { imageToBeModified }, new Guid[] { tagToBeModified }); untaggedImages = client.GetUntaggedImagesAsync(project.ProjectId).Result; Assert.Equal(1, untaggedImages.Count); var imageTags = new ImageTagCreateEntry(imageToBeModified, tagToBeModified); var result = client.CreateImageTagsAsync(project.ProjectId, new ImageTagCreateBatch(new ImageTagCreateEntry[] { imageTags })).Result; Assert.Equal(1, result.Created.Count); Assert.Equal(imageToBeModified, result.Created[0].ImageId); Assert.Equal(tagToBeModified, result.Created[0].TagId); } } }
public async void TrainProject() { using (MockContext context = MockContext.Start(this.GetType().Name)) { HttpMockServer.Initialize(this.GetType().Name, "TrainProject", RecorderMode); var projectId = CreateTrainedImageClassificationProject(); ICustomVisionTrainingClient client = GetTrainingClient(); // Remove the last iteration so we can retrain var iterations = await client.GetIterationsAsync(projectId); var iterationToDelete = iterations[iterations.Count - 1]; iterationToDelete.IsDefault = false; await client.UpdateIterationAsync(projectId, iterationToDelete.Id, iterationToDelete); await client.DeleteIterationAsync(projectId, iterationToDelete.Id); // Need to ensure we wait 1 second between training calls from the previous project. Or // We get 429s. Thread.Sleep(1000); var trainedIteration = await client.TrainProjectAsync(projectId); Assert.NotEqual(iterationToDelete.Name, trainedIteration.Name); Assert.NotEqual(iterationToDelete.Id, trainedIteration.Id); Assert.NotEqual(Guid.Empty, trainedIteration.Id); Assert.False(trainedIteration.IsDefault); Assert.True("Staging" == trainedIteration.Status || "Training" == trainedIteration.Status); Assert.False(trainedIteration.Exportable); await client.DeleteProjectAsync(projectId); } }
public async void ObjDetectionPrediction() { using (MockContext context = MockContext.Start(this.GetType())) { HttpMockServer.Initialize(this.GetType(), "ObjDetectionPrediction", RecorderMode); using (var project = CreateTrainedObjDetectionProject()) using (ICustomVisionTrainingClient client = BaseTests.GetTrainingClient()) using (FileStream stream = new FileStream(Path.Combine("TestImages", "od_test_image.jpg"), FileMode.Open)) { var imageResult = await client.QuickTestImageAsync(project.ProjectId, stream, project.IterationId); Assert.NotEqual(Guid.Empty, imageResult.Id); Assert.NotEqual(Guid.Empty, imageResult.Iteration); Assert.Equal(project.ProjectId, imageResult.Project); Assert.NotEqual(0, imageResult.Predictions.Count); Assert.InRange(imageResult.Predictions[0].Probability, 0.5, 1); Assert.NotNull(imageResult.Predictions[0].BoundingBox); Assert.InRange(imageResult.Predictions[0].BoundingBox.Left, 0, 1); Assert.InRange(imageResult.Predictions[0].BoundingBox.Top, 0, 1); Assert.InRange(imageResult.Predictions[0].BoundingBox.Width, 0, 1); Assert.InRange(imageResult.Predictions[0].BoundingBox.Height, 0, 1); } } }
public async void ImageCounts() { using (MockContext context = MockContext.Start(this.GetType())) { HttpMockServer.Initialize(this.GetType(), "ImageCounts", RecorderMode); using (var project = CreateTrainedImageClassificationProject()) { ICustomVisionTrainingClient client = BaseTests.GetTrainingClient(); var taggedImageCount = await client.GetTaggedImageCountAsync(project.ProjectId); Assert.Equal(10, taggedImageCount); var untaggedImageCount = await client.GetUntaggedImageCountAsync(project.ProjectId); Assert.Equal(0, untaggedImageCount); var iterationId = (await client.GetIterationsAsync(project.ProjectId))[0].Id; var imagePerfCount = await client.GetImagePerformanceCountAsync(project.ProjectId, iterationId); Assert.Equal(2, imagePerfCount); } } }
public async void CreateImageFromUrl() { string imageUrl = "https://raw.githubusercontent.com/Microsoft/Cognitive-CustomVision-Windows/master/Samples/Images/Hemlock/hemlock_1.jpg"; using (MockContext context = MockContext.Start(this.GetType())) { HttpMockServer.Initialize(this.GetType(), "CreateImageFromUrl", RecorderMode); using (ICustomVisionTrainingClient client = GetTrainingClient()) { var newProject = client.CreateProjectAsync(projName, projDescription, ProjectBuilderHelper.FoodDomain).Result; var tag = client.CreateTagAsync(newProject.Id, tagName).Result; var urlImages = new ImageUrlCreateEntry[] { new ImageUrlCreateEntry(imageUrl) }; var tags = new Guid[] { tag.Id }; var imageCreatedFromUrl = client.CreateImagesFromUrlsAsync(newProject.Id, new ImageUrlCreateBatch(urlImages, tags)).Result; Assert.True(imageCreatedFromUrl.IsBatchSuccessful); Assert.Equal(1, imageCreatedFromUrl.Images.Count); Assert.Equal(imageUrl, imageCreatedFromUrl.Images[0].SourceUrl); Assert.Equal("OK", imageCreatedFromUrl.Images[0].Status); Assert.NotEqual(Guid.Empty, imageCreatedFromUrl.Images[0].Image.Id); Assert.NotEqual(0, imageCreatedFromUrl.Images[0].Image.Width); Assert.NotEqual(0, imageCreatedFromUrl.Images[0].Image.Height); Assert.NotEmpty(imageCreatedFromUrl.Images[0].Image.OriginalImageUri); Assert.NotEmpty(imageCreatedFromUrl.Images[0].Image.ResizedImageUri); Assert.NotEmpty(imageCreatedFromUrl.Images[0].Image.ThumbnailUri); await client.DeleteProjectAsync(newProject.Id); } } }
public async void QuerySuggestedImageCount() { using (MockContext context = MockContext.Start(this.GetType())) { HttpMockServer.Initialize(this.GetType(), "QuerySuggestedImageCount", RecorderMode); using (var project = CreateTrainedImageClassificationProject()) using (ICustomVisionTrainingClient client = BaseTests.GetTrainingClient()) { // Add an untagged image we expect to be classified as tag1 var images = new ImageFileCreateEntry[] { new ImageFileCreateEntry("suggest_ic.jpg", File.ReadAllBytes(Path.Combine("TestImages", "suggest_ic.jpg"))) }; var imageResult = await client.CreateImagesFromFilesAsync(project.ProjectId, new ImageFileCreateBatch(images)); Assert.True(imageResult.IsBatchSuccessful); Assert.Equal(1, imageResult.Images.Count); // Ask for suggestions var suggestions = client.SuggestTagsAndRegions(project.ProjectId, project.IterationId, new Guid[] { imageResult.Images[0].Image.Id }); // Validate result Assert.Equal(1, suggestions.Count); var tags = await client.GetTagsAsync(project.ProjectId, project.IterationId); // We expect to get Tag1 as the primary suggestion, so query with a high prediction var countMapping = await client.QuerySuggestedImageCountAsync(project.ProjectId, project.IterationId, new TagFilter() { Threshold = 0.75, TagIds = new Guid[] { tags[0].Id } }); Assert.Equal(1, countMapping.Count); Assert.Equal(1, countMapping[tags[0].Id.ToString()]); // We expect to get Tag2 to have a low probabilty, make sure we don't find it with a high threshold countMapping = await client.QuerySuggestedImageCountAsync(project.ProjectId, project.IterationId, new TagFilter() { Threshold = 0.75, TagIds = new Guid[] { tags[1].Id } }); Assert.Equal(1, countMapping.Count); Assert.Equal(0, countMapping[tags[1].Id.ToString()]); // Get results for all tags with a high threshold. countMapping = await client.QuerySuggestedImageCountAsync(project.ProjectId, project.IterationId, new TagFilter() { Threshold = 0.75, TagIds = tags.Select(t => t.Id).ToList() }); Assert.Equal(2, countMapping.Count); Assert.Equal(1, countMapping[tags[0].Id.ToString()]); Assert.Equal(0, countMapping[tags[1].Id.ToString()]); } } }
public static Iteration CreateTrainedImageClassificationProject(ICustomVisionTrainingClient client, string predictionResourceId, Guid?domain = null) { #if RECORD_MODE var projName = ProjectPrefix + Guid.NewGuid().ToString(); // Create a project var projDomain = domain.HasValue ? domain : GeneralDomain; var project = client.CreateProject(projName, null, projDomain); // Create two tags var tagOne = client.CreateTag(project.Id, "Tag1"); var tagTwo = client.CreateTag(project.Id, "Tag2"); // Add five images for tag 1 var imagePath = Path.Combine("TestImages", "tag1"); var imageFileEntries = new List <ImageFileCreateEntry>(); foreach (var fileName in Directory.EnumerateFiles(imagePath)) { imageFileEntries.Add(new ImageFileCreateEntry(fileName, File.ReadAllBytes(fileName))); } client.CreateImagesFromFiles(project.Id, new ImageFileCreateBatch(imageFileEntries, new List <Guid>(new Guid[] { tagOne.Id }))); // Add five images for tag 2 imagePath = Path.Combine("TestImages", "tag2"); imageFileEntries = new List <ImageFileCreateEntry>(); foreach (var fileName in Directory.EnumerateFiles(imagePath)) { imageFileEntries.Add(new ImageFileCreateEntry(fileName, File.ReadAllBytes(fileName))); } client.CreateImagesFromFiles(project.Id, new ImageFileCreateBatch(imageFileEntries, new List <Guid>(new Guid[] { tagTwo.Id }))); // Train var iteration = client.TrainProject(project.Id); while (iteration.Status != "Completed") { Thread.Sleep(1000); iteration = client.GetIteration(project.Id, iteration.Id); } Assert.True(client.PublishIteration(project.Id, iteration.Id, ClassificationPublishName, predictionResourceId)); // Get latest iteration iteration = client.GetIteration(project.Id, iteration.Id); // Make one prediction string imageUrl = "https://raw.githubusercontent.com/Microsoft/Cognitive-CustomVision-Windows/master/Samples/Images/Test/test_image.jpg"; client.QuickTestImageUrl(project.Id, new ImageUrl(imageUrl), iteration.Id); // Flush and re-init so we don't get all of the test setup in the session. HttpMockServer.Flush(); HttpMockServer.Initialize(HttpMockServer.CallerIdentity, HttpMockServer.TestIdentity, HttpRecorderMode.Record); return(iteration); #else return(null); #endif }
public async void RegionManipulation() { using (MockContext context = MockContext.Start(this.GetType())) { HttpMockServer.Initialize(this.GetType(), "RegionManipulation", RecorderMode); using (var project = CreateTrainedObjDetectionProject()) using (ICustomVisionTrainingClient client = BaseTests.GetTrainingClient()) { var existingImage = (await client.GetTaggedImagesAsync(project.ProjectId)).First(); Assert.NotNull(existingImage); var testTag = await client.CreateTagAsync(project.ProjectId, tagName); var proposal = await client.GetImageRegionProposalsAsync(project.ProjectId, existingImage.Id); Assert.True(proposal.Proposals.Count > 0); Assert.Equal(project.ProjectId, proposal.ProjectId); Assert.Equal(existingImage.Id, proposal.ImageId); Assert.InRange(proposal.Proposals[0].Confidence, 0, 1); var bbox = proposal.Proposals[0].BoundingBox; List <ImageRegionCreateEntry> newRegions = new List <ImageRegionCreateEntry>(); newRegions.Add(new ImageRegionCreateEntry(existingImage.Id, testTag.Id, bbox.Left, bbox.Top, bbox.Width, bbox.Height)); var regions = await client.CreateImageRegionsAsync(project.ProjectId, new ImageRegionCreateBatch(newRegions)); Assert.Equal(1, regions.Created.Count); Assert.Equal(0, regions.Duplicated.Count); Assert.Equal(0, regions.Exceeded.Count); Assert.Equal(bbox.Top, regions.Created[0].Top); Assert.Equal(bbox.Left, regions.Created[0].Left); Assert.Equal(bbox.Width, regions.Created[0].Width); Assert.Equal(bbox.Height, regions.Created[0].Height); Assert.Equal(existingImage.Id, regions.Created[0].ImageId); Assert.Equal(testTag.Id, regions.Created[0].TagId); Assert.NotEqual(Guid.Empty, regions.Created[0].RegionId); var image = await client.GetImagesByIdsAsync(project.ProjectId, new List <Guid>(new Guid[] { existingImage.Id })); Assert.Equal(1, image.Count); Assert.Equal(2, image[0].Regions.Count); await client.DeleteImageRegionsAsync(project.ProjectId, new List <Guid>(new Guid[] { regions.Created[0].RegionId })); image = await client.GetImagesByIdsAsync(project.ProjectId, new List <Guid>(new Guid[] { existingImage.Id })); Assert.Equal(1, image.Count); Assert.Equal(1, image[0].Regions.Count); await client.DeleteTagAsync(project.ProjectId, testTag.Id); } } }
public static ICustomVisionTrainingClient GetTrainingClient(bool addHandler = true) { ICustomVisionTrainingClient client = (addHandler) ? new CustomVisionTrainingClient(handlers: HttpMockServer.CreateInstance()) : new CustomVisionTrainingClient(); client.ApiKey = TrainingKey; client.Endpoint = Endpoint; return(client); }
public static ICustomVisionTrainingClient GetTrainingClient(bool addHandler = true) { var credentials = new ApiKeyServiceClientCredentials(TrainingKey); ICustomVisionTrainingClient client = (addHandler) ? new CustomVisionTrainingClient(credentials, handlers: HttpMockServer.CreateInstance()) : new CustomVisionTrainingClient(credentials); client.Endpoint = Endpoint; return(client); }
public static async Task <TrainingModels::Export> ExportIteration( this ICustomVisionTrainingClient trainingApi, Guid projectId, Guid iterationId, string flavor = "onnx12", int timeoutInSecond = 30) { TimeSpan timeout = TimeSpan.FromSeconds(timeoutInSecond); TrainingModels::Export exportIteration = null; try { exportIteration = exportIteration = string.IsNullOrEmpty(flavor) ? await trainingApi.ExportIterationAsync(projectId, iterationId, platform) : await trainingApi.ExportIterationAsync(projectId, iterationId, platform, flavor); } catch (HttpOperationException ex) { string exceptionContent = ex?.Response?.Content ?? string.Empty; if (!exceptionContent.Contains("BadRequestExportAlreadyInProgress")) { throw ex; } } DateTime startTime = DateTime.Now; while (true) { IList <TrainingModels::Export> exports = await trainingApi.GetExportsAsync(projectId, iterationId); exportIteration = string.IsNullOrEmpty(flavor) ? exports?.FirstOrDefault(x => string.Equals(x.Platform, platform, StringComparison.OrdinalIgnoreCase)) : exports?.FirstOrDefault(x => string.Equals(x.Platform, platform, StringComparison.OrdinalIgnoreCase) && string.Equals(x.Flavor, flavor, StringComparison.OrdinalIgnoreCase)); if (exportIteration?.Status == "Exporting") { await Task.Delay(1000); } else { break; } if (DateTime.Now - startTime > timeout) { throw new TimeoutException("The operation couldn't be completed due to a timeout."); } } return(exportIteration); }
public void Dispose() { #if RECORD_MODE ICustomVisionTrainingClient client = BaseTests.GetTrainingClient(false); foreach (var iter in client.GetIterations(this.ProjectId)) { if (!string.IsNullOrEmpty(iter.PublishName)) { client.UnpublishIteration(this.ProjectId, iter.Id); } } client.DeleteProject(this.ProjectId); #endif }
public async void GetUntaggedImages() { using (MockContext context = MockContext.Start(this.GetType())) { HttpMockServer.Initialize(this.GetType(), "GetUntaggedImages", RecorderMode); using (var project = CreateTrainedImageClassificationProject()) { ICustomVisionTrainingClient client = BaseTests.GetTrainingClient(); var images = await client.GetUntaggedImagesAsync(project.ProjectId); Assert.Equal(0, images.Count); } } }
public async void GetUntaggedImages() { using (MockContext context = MockContext.Start(this.GetType().Name)) { HttpMockServer.Initialize(this.GetType().Name, "GetUntaggedImages", RecorderMode); var projectId = CreateTrainedImageClassificationProject(); ICustomVisionTrainingClient client = GetTrainingClient(); var images = client.GetUntaggedImagesAsync(projectId).Result; Assert.Equal(0, images.Count); await client.DeleteProjectAsync(projectId); } }
public async void CreateDeleteProjectWithDomain() { using (MockContext context = MockContext.Start(this.GetType().Name)) { HttpMockServer.Initialize(this.GetType().Name, "CreateDeleteProjectWithDomain", RecorderMode); ICustomVisionTrainingClient client = GetTrainingClient(); var newProject = await client.CreateProjectAsync(projName, projDescription, FoodDomain); Assert.Contains(projName, newProject.Name); Assert.Equal(projDescription, newProject.Description); Assert.Equal(FoodDomain, newProject.Settings.DomainId); Assert.NotEqual(Guid.Empty, newProject.Id); await client.DeleteProjectAsync(newProject.Id); } }
public static void CleanUpOldProjects(ICustomVisionTrainingClient client) { foreach (var project in client.GetProjects()) { if (project.Name.StartsWith(ProjectPrefix)) { System.Console.WriteLine("Cleaning old project: " + project.Name); foreach (var iter in client.GetIterations(project.Id)) { if (!string.IsNullOrEmpty(iter.PublishName)) { client.UnpublishIteration(project.Id, iter.Id); } } client.DeleteProject(project.Id); } } }
public static async Task <TrainingModels::Export> ExportIteration(this ICustomVisionTrainingClient trainingApi, Guid projectId, Guid iterationId, int timeoutInSecond = 30) { TimeSpan timeout = TimeSpan.FromSeconds(timeoutInSecond); TrainingModels::Export exportIteration = null; try { exportIteration = await trainingApi.ExportIterationAsync(projectId, iterationId, platform); } catch (TrainingModels::CustomVisionErrorException ex) { if (ex.Body.Code != TrainingModels::CustomVisionErrorCodes.BadRequestExportAlreadyInProgress) { throw ex; } } DateTime startTime = DateTime.Now; while (true) { IList <TrainingModels::Export> exports = await trainingApi.GetExportsAsync(projectId, iterationId); exportIteration = exports?.FirstOrDefault(x => string.Equals(x.Platform, platform, StringComparison.OrdinalIgnoreCase)); if (exportIteration?.Status == "Exporting") { await Task.Delay(1000); } else { break; } if (DateTime.Now - startTime > timeout) { throw new TimeoutException("The operation couldn't be completed due to a timeout."); } } return(exportIteration); }
public async void TrainAndPublishProject() { using (MockContext context = MockContext.Start(this.GetType())) { HttpMockServer.Initialize(this.GetType(), "TrainAndPublishProject", RecorderMode); using (var project = CreateTrainedImageClassificationProject()) { ICustomVisionTrainingClient client = BaseTests.GetTrainingClient(); // Remove the last trained iteration so we can retrain var iterationToDelete = client.GetIteration(project.ProjectId, project.IterationId); var originalPublishName = iterationToDelete.PublishName; await client.UnpublishIterationAsync(project.ProjectId, iterationToDelete.Id); await client.DeleteIterationAsync(project.ProjectId, iterationToDelete.Id); // Need to ensure we wait 1 second between training calls from the previous project.Or // We get 429s. Thread.Sleep(1000); var trainedIteration = await client.TrainProjectAsync(project.ProjectId); Assert.NotEqual(iterationToDelete.Name, trainedIteration.Name); Assert.NotEqual(iterationToDelete.Id, trainedIteration.Id); Assert.NotEqual(Guid.Empty, trainedIteration.Id); Assert.True("Staging" == trainedIteration.Status || "Training" == trainedIteration.Status); Assert.False(trainedIteration.Exportable); Assert.Null(trainedIteration.PublishName); // Wait for training to complete. while (trainedIteration.Status != "Completed") { Thread.Sleep(1000); trainedIteration = client.GetIteration(project.ProjectId, trainedIteration.Id); } // Verify we can republish using same name Assert.True(client.PublishIteration(project.ProjectId, trainedIteration.Id, originalPublishName, BaseTests.PredictionResourceId)); project.IterationId = trainedIteration.Id; } } }
public async void QuerySuggestedImages() { using (MockContext context = MockContext.Start(this.GetType())) { HttpMockServer.Initialize(this.GetType(), "QuerySuggestedImages", RecorderMode); using (var project = CreateTrainedImageClassificationProject()) using (ICustomVisionTrainingClient client = BaseTests.GetTrainingClient()) { // Add an untagged image we expect to be classified as tag1 var images = new ImageFileCreateEntry[] { new ImageFileCreateEntry("suggest_ic.jpg", File.ReadAllBytes(Path.Combine("TestImages", "suggest_ic.jpg"))) }; var imageResult = await client.CreateImagesFromFilesAsync(project.ProjectId, new ImageFileCreateBatch(images)); Assert.True(imageResult.IsBatchSuccessful); Assert.Equal(1, imageResult.Images.Count); // Ask for suggestions var suggestions = client.SuggestTagsAndRegions(project.ProjectId, project.IterationId, new Guid[] { imageResult.Images[0].Image.Id }); // Validate result Assert.Equal(1, suggestions.Count); // Get tags and build a query var tags = await client.GetTagsAsync(project.ProjectId, project.IterationId); var query = new SuggestedTagAndRegionQueryToken() { MaxCount = 10, TagIds = tags.Select(t => t.Id).ToList(), Threshold = 0 }; // This will return all suggested images (1 in this case) var suggestedImages = await client.QuerySuggestedImagesAsync(project.ProjectId, project.IterationId, query); Assert.Equal(1, suggestedImages.Results.Count); } } }