public async void TagRetrieval()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
HttpMockServer.Initialize(this.GetType(), "TagRetrieval", RecorderMode);
using (ICustomVisionTrainingClient client = GetTrainingClient())
{
var newProject = await client.CreateProjectAsync("TagRetrievalTest", projDescription, ProjectBuilderHelper.GeneralDomain);
var numTags = 4;
for (var i = 0; i < numTags; i++)
{
await client.CreateTagAsync(newProject.Id, "Tag #" + i.ToString(), string.Empty);
}
var tagList = await client.GetTagsAsync(newProject.Id);
Assert.Equal(numTags, tagList.Count);
foreach (var tag in tagList)
{
Assert.NotEqual(Guid.Empty, tag.Id);
Assert.Contains("Tag #", tag.Name);
Assert.Null(tag.Description);
Assert.Equal(0, tag.ImageCount);
}
await client.DeleteProjectAsync(newProject.Id);
}
}
}
public async void CreateImagesFromFiles()
{
var dataFileName = "hemlock_1.jpg";
using (MockContext context = MockContext.Start(this.GetType().Name))
{
HttpMockServer.Initialize(this.GetType().Name, "CreateImagesFromFiles", RecorderMode);
ICustomVisionTrainingClient client = GetTrainingClient();
var newProject = client.CreateProjectAsync(projName, projDescription, FoodDomain).Result;
var tag = client.CreateTagAsync(newProject.Id, tagName).Result;
var fileName = Path.Combine("TestImages", "tag1", dataFileName);
var fileImages = new ImageFileCreateEntry[] { new ImageFileCreateEntry(dataFileName, File.ReadAllBytes(fileName)) };
var tags = new Guid[] { tag.Id };
var imageCreatedFromFile = client.CreateImagesFromFilesAsync(newProject.Id, new ImageFileCreateBatch(fileImages, tags)).Result;
Assert.True(imageCreatedFromFile.IsBatchSuccessful);
Assert.Equal(1, imageCreatedFromFile.Images.Count);
Assert.Contains(dataFileName, imageCreatedFromFile.Images[0].SourceUrl);
Assert.Equal("OK", imageCreatedFromFile.Images[0].Status);
Assert.NotEqual(Guid.Empty, imageCreatedFromFile.Images[0].Image.Id);
Assert.NotEqual(0, imageCreatedFromFile.Images[0].Image.Width);
Assert.NotEqual(0, imageCreatedFromFile.Images[0].Image.Height);
Assert.NotEmpty(imageCreatedFromFile.Images[0].Image.OriginalImageUri);
Assert.NotEmpty(imageCreatedFromFile.Images[0].Image.ResizedImageUri);
Assert.NotEmpty(imageCreatedFromFile.Images[0].Image.ThumbnailUri);
await client.DeleteProjectAsync(newProject.Id);
}
}
public static async Task <ProjectIteration[]> GetProjectIterations(ICustomVisionTrainingClient trainingClient, string predictionResourceId, Guid[] projects)
{
var result = new List <ProjectIteration>();
foreach (var project in projects)
{
var iterations = await trainingClient.GetIterationsAsync(project);
var projectEntity = await trainingClient.GetProjectAsync(project);
var domain = await trainingClient.GetDomainAsync(projectEntity.Settings.DomainId);
var iteration = iterations.Where(i => i.Status == "Completed").OrderByDescending(i => i.TrainedAt.Value).FirstOrDefault();
if (iteration != null)
{
if (string.IsNullOrEmpty(iteration.PublishName))
{
await trainingClient.PublishIterationAsync(project, iteration.Id, publishName : iteration.Id.ToString(), predictionId : predictionResourceId);
iteration = await trainingClient.GetIterationAsync(project, iteration.Id);
}
result.Add(new ProjectIteration()
{
Project = project, Iteration = iteration.Id, ProjectName = projectEntity.Name, PublishedName = iteration.PublishName, IsObjectDetection = domain.Type == "ObjectDetection"
});
}
}
return(result.ToArray());
}
public async void CreateImagesFromData()
{
var dataFileName = "hemlock_1.jpg";
using (MockContext context = MockContext.Start(this.GetType()))
{
HttpMockServer.Initialize(this.GetType(), "CreateImagesFromData", RecorderMode);
using (ICustomVisionTrainingClient client = GetTrainingClient())
{
var newProject = client.CreateProjectAsync(projName, projDescription, ProjectBuilderHelper.FoodDomain).Result;
var tag = client.CreateTagAsync(newProject.Id, tagName).Result;
using (FileStream stream = new FileStream(Path.Combine("TestImages", "tag1", dataFileName), FileMode.Open))
{
var imageCreatedFromData = client.CreateImagesFromDataAsync(newProject.Id, stream, new Guid[] { tag.Id }).Result;
Assert.True(imageCreatedFromData.IsBatchSuccessful);
Assert.Equal(1, imageCreatedFromData.Images.Count);
Assert.Contains(dataFileName, imageCreatedFromData.Images[0].SourceUrl);
Assert.Equal("OK", imageCreatedFromData.Images[0].Status);
Assert.NotEqual(Guid.Empty, imageCreatedFromData.Images[0].Image.Id);
Assert.NotEqual(0, imageCreatedFromData.Images[0].Image.Width);
Assert.NotEqual(0, imageCreatedFromData.Images[0].Image.Height);
Assert.NotEmpty(imageCreatedFromData.Images[0].Image.OriginalImageUri);
Assert.NotEmpty(imageCreatedFromData.Images[0].Image.ResizedImageUri);
Assert.NotEmpty(imageCreatedFromData.Images[0].Image.ThumbnailUri);
}
await client.DeleteProjectAsync(newProject.Id);
}
}
}
public async void GetImagesByIds()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
HttpMockServer.Initialize(this.GetType(), "GetImagesByIds", RecorderMode);
using (var project = CreateTrainedImageClassificationProject())
{
ICustomVisionTrainingClient client = BaseTests.GetTrainingClient();
var tags = await client.GetTagsAsync(project.ProjectId);
var imagesForTag = await client.GetTaggedImagesAsync(project.ProjectId, null, new List <Guid>(new Guid[] { tags[0].Id }));
Assert.Equal(5, imagesForTag.Count);
var images = await client.GetImagesByIdsAsync(project.ProjectId, imagesForTag.Select(img => img.Id).ToList());
Assert.Equal(5, images.Count);
foreach (var image in images)
{
Assert.Equal(1, image.Tags.Count);
var tag = image.Tags[0];
Assert.Equal(tags[0].Id, tag.TagId);
Assert.Equal(tags[0].Name, tag.TagName);
Assert.NotNull(image.ThumbnailUri);
Assert.NotEmpty(image.OriginalImageUri);
Assert.NotEmpty(image.ResizedImageUri);
Assert.True(image.Width > 0);
Assert.True(image.Height > 0);
}
}
}
}
public async void CreateUpdateDeleteProject()
{
var updatedProjName = "Another Name";
var updatedProjDescription = "Updated Project Description";
using (MockContext context = MockContext.Start(this.GetType()))
{
HttpMockServer.Initialize(this.GetType(), "CreateUpdateDeleteProject", RecorderMode);
using (ICustomVisionTrainingClient client = GetTrainingClient())
{
var newProject = client.CreateProjectAsync(projName, projDescription).Result;
Assert.Contains(projName, newProject.Name);
Assert.Equal(projDescription, newProject.Description);
Assert.NotEqual(Guid.Empty, newProject.Id);
var updatedProject = client.UpdateProjectAsync(newProject.Id, new Project()
{
Name = updatedProjName,
Description = updatedProjDescription,
}).Result;
Assert.Equal(updatedProjName, updatedProject.Name);
Assert.Equal(updatedProjDescription, updatedProject.Description);
Assert.Equal(newProject.Id, updatedProject.Id);
Assert.Equal(newProject.Settings.DomainId, updatedProject.Settings.DomainId);
await client.DeleteProjectAsync(newProject.Id);
}
}
}
public async void ProjectRetrieval()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
HttpMockServer.Initialize(this.GetType(), "ProjectRetrieval", RecorderMode);
using (ICustomVisionTrainingClient client = GetTrainingClient())
{
var newProject = await client.CreateProjectAsync(projName, projDescription, ProjectBuilderHelper.FoodDomain);
var projects = await client.GetProjectsAsync();
Assert.True(projects.Count > 0);
Assert.Contains(projects, p => p.Id == newProject.Id);
var firstProject = await client.GetProjectAsync(projects[0].Id);
Assert.Equal(projects[0].Id, firstProject.Id);
Assert.Equal(projects[0].Name, firstProject.Name);
Assert.Equal(projects[0].Description, firstProject.Description);
Assert.Equal(projects[0].Settings.DomainId, firstProject.Settings.DomainId);
Assert.Equal(projects[0].Settings.TargetExportPlatforms.Count, firstProject.Settings.TargetExportPlatforms.Count);
Assert.Equal(projects[0].DrModeEnabled, firstProject.DrModeEnabled);
await client.DeleteProjectAsync(newProject.Id);
}
}
}
public async void GetIterationPerformance()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
HttpMockServer.Initialize(this.GetType(), "GetIterationPerformance", RecorderMode);
using (var project = CreateTrainedImageClassificationProject())
{
ICustomVisionTrainingClient client = BaseTests.GetTrainingClient();
var iterations = await client.GetIterationsAsync(project.ProjectId);
Assert.True(iterations.Count > 0);
var iterationPerf = await client.GetIterationPerformanceAsync(project.ProjectId, iterations[iterations.Count - 1].Id, 0.9);
Assert.Equal(1, iterationPerf.Recall);
Assert.Equal(0, iterationPerf.RecallStdDeviation);
Assert.Equal(1, iterationPerf.Precision);
Assert.Equal(0, iterationPerf.PrecisionStdDeviation);
Assert.Equal(2, iterationPerf.PerTagPerformance.Count);
Assert.Equal("Tag1", iterationPerf.PerTagPerformance[0].Name);
Assert.Equal(1, iterationPerf.PerTagPerformance[0].Recall);
Assert.Equal(0, iterationPerf.PerTagPerformance[0].RecallStdDeviation);
Assert.Equal(1, iterationPerf.PerTagPerformance[0].Precision);
Assert.Equal(0, iterationPerf.PerTagPerformance[0].PrecisionStdDeviation);
Assert.Equal("Tag2", iterationPerf.PerTagPerformance[1].Name);
Assert.Equal(1, iterationPerf.PerTagPerformance[1].Recall);
Assert.Equal(0, iterationPerf.PerTagPerformance[1].RecallStdDeviation);
Assert.Equal(1, iterationPerf.PerTagPerformance[1].Precision);
Assert.Equal(0, iterationPerf.PerTagPerformance[1].PrecisionStdDeviation);
}
}
}
public async void QuickTests()
{
var dataFileName = "test_image.jpg";
using (MockContext context = MockContext.Start(this.GetType()))
{
HttpMockServer.Initialize(this.GetType(), "QuickTests", RecorderMode);
using (var project = CreateTrainedImageClassificationProject())
using (ICustomVisionTrainingClient client = BaseTests.GetTrainingClient())
{
using (FileStream stream = new FileStream(Path.Combine("TestImages", dataFileName), FileMode.Open))
{
var imageResult = await client.QuickTestImageAsync(project.ProjectId, stream, project.IterationId);
Assert.NotEqual(Guid.Empty, imageResult.Id);
Assert.NotEqual(Guid.Empty, imageResult.Iteration);
Assert.Equal(project.ProjectId, imageResult.Project);
Assert.NotEqual(0, imageResult.Predictions.Count);
Assert.InRange(imageResult.Predictions[0].Probability, 0.9, 1);
Assert.Equal("Tag1", imageResult.Predictions[0].TagName);
}
string imageUrl = "https://raw.githubusercontent.com/Microsoft/Cognitive-CustomVision-Windows/master/Samples/Images/Test/test_image.jpg";
var urlResult = await client.QuickTestImageUrlAsync(project.ProjectId, new ImageUrl(imageUrl), project.IterationId);
Assert.NotEqual(Guid.Empty, urlResult.Id);
Assert.NotEqual(Guid.Empty, urlResult.Iteration);
Assert.Equal(project.ProjectId, urlResult.Project);
Assert.NotEqual(0, urlResult.Predictions.Count);
Assert.InRange(urlResult.Predictions[0].Probability, 0.9, 1);
Assert.Equal("Tag1", urlResult.Predictions[0].TagName);
}
}
}
public async void QueryPredictionResults()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
HttpMockServer.Initialize(this.GetType(), "QueryPredictionResults", RecorderMode);
using (var project = CreateTrainedImageClassificationProject())
{
ICustomVisionTrainingClient client = BaseTests.GetTrainingClient();
var token = new PredictionQueryToken()
{
OrderBy = "Newest",
};
var result = await client.QueryPredictionsAsync(project.ProjectId, token);
Assert.NotEqual(0, result.Results.Count);
foreach (var prediction in result.Results)
{
Assert.Equal(project.ProjectId, prediction.Project);
Assert.NotEqual(Guid.Empty, prediction.Id);
Assert.NotEqual(Guid.Empty, prediction.Iteration);
Assert.NotEmpty(prediction.ThumbnailUri);
Assert.NotEmpty(prediction.OriginalImageUri);
Assert.NotEmpty(prediction.ResizedImageUri);
Assert.NotEqual(0, prediction.Predictions.Count);
}
}
}
}
public async void ExportTests()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
HttpMockServer.Initialize(this.GetType(), "ExportTests", RecorderMode);
using (var project = CreateTrainedImageClassificationProject(ProjectBuilderHelper.ExportableDomain))
{
ICustomVisionTrainingClient client = BaseTests.GetTrainingClient();
var iterations = await client.GetIterationsAsync(project.ProjectId);
var exportableIteration = iterations.FirstOrDefault(e => e.Exportable);
Assert.NotNull(exportableIteration);
var export = await client.ExportIterationAsync(project.ProjectId, exportableIteration.Id, ExportPlatform.TensorFlow);
Assert.Equal("Exporting", export.Status);
Assert.Null(export.DownloadUri);
Assert.Equal(ExportPlatform.TensorFlow, export.Platform);
Assert.False(export.NewerVersionAvailable);
while (export.Status != "Done")
{
var exports = await client.GetExportsAsync(project.ProjectId, exportableIteration.Id);
Assert.Equal(1, exports.Count);
export = exports.Where(e => e.Platform == ExportPlatform.TensorFlow).FirstOrDefault();
Assert.NotNull(export);
Thread.Sleep(1000);
}
Assert.NotEmpty(export.DownloadUri);
}
}
}
public async void ImageTagManipulation()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
HttpMockServer.Initialize(this.GetType(), "ImageTagManipulation", RecorderMode);
using (var project = CreateTrainedImageClassificationProject())
{
ICustomVisionTrainingClient client = BaseTests.GetTrainingClient();
var untaggedImages = client.GetUntaggedImagesAsync(project.ProjectId).Result;
Assert.Equal(0, untaggedImages.Count);
var taggedImages = client.GetTaggedImagesAsync(project.ProjectId).Result;
Assert.Equal(10, taggedImages.Count);
var imageToBeModified = taggedImages[0].Id;
var tagToBeModified = taggedImages[0].Tags[0].TagId;
await client.DeleteImageTagsAsync(project.ProjectId, new Guid[] { imageToBeModified }, new Guid[] { tagToBeModified });
untaggedImages = client.GetUntaggedImagesAsync(project.ProjectId).Result;
Assert.Equal(1, untaggedImages.Count);
var imageTags = new ImageTagCreateEntry(imageToBeModified, tagToBeModified);
var result = client.CreateImageTagsAsync(project.ProjectId, new ImageTagCreateBatch(new ImageTagCreateEntry[] { imageTags })).Result;
Assert.Equal(1, result.Created.Count);
Assert.Equal(imageToBeModified, result.Created[0].ImageId);
Assert.Equal(tagToBeModified, result.Created[0].TagId);
}
}
}
public async void TrainProject()
{
using (MockContext context = MockContext.Start(this.GetType().Name))
{
HttpMockServer.Initialize(this.GetType().Name, "TrainProject", RecorderMode);
var projectId = CreateTrainedImageClassificationProject();
ICustomVisionTrainingClient client = GetTrainingClient();
// Remove the last iteration so we can retrain
var iterations = await client.GetIterationsAsync(projectId);
var iterationToDelete = iterations[iterations.Count - 1];
iterationToDelete.IsDefault = false;
await client.UpdateIterationAsync(projectId, iterationToDelete.Id, iterationToDelete);
await client.DeleteIterationAsync(projectId, iterationToDelete.Id);
// Need to ensure we wait 1 second between training calls from the previous project. Or
// We get 429s.
Thread.Sleep(1000);
var trainedIteration = await client.TrainProjectAsync(projectId);
Assert.NotEqual(iterationToDelete.Name, trainedIteration.Name);
Assert.NotEqual(iterationToDelete.Id, trainedIteration.Id);
Assert.NotEqual(Guid.Empty, trainedIteration.Id);
Assert.False(trainedIteration.IsDefault);
Assert.True("Staging" == trainedIteration.Status || "Training" == trainedIteration.Status);
Assert.False(trainedIteration.Exportable);
await client.DeleteProjectAsync(projectId);
}
}
public async void ObjDetectionPrediction()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
HttpMockServer.Initialize(this.GetType(), "ObjDetectionPrediction", RecorderMode);
using (var project = CreateTrainedObjDetectionProject())
using (ICustomVisionTrainingClient client = BaseTests.GetTrainingClient())
using (FileStream stream = new FileStream(Path.Combine("TestImages", "od_test_image.jpg"), FileMode.Open))
{
var imageResult = await client.QuickTestImageAsync(project.ProjectId, stream, project.IterationId);
Assert.NotEqual(Guid.Empty, imageResult.Id);
Assert.NotEqual(Guid.Empty, imageResult.Iteration);
Assert.Equal(project.ProjectId, imageResult.Project);
Assert.NotEqual(0, imageResult.Predictions.Count);
Assert.InRange(imageResult.Predictions[0].Probability, 0.5, 1);
Assert.NotNull(imageResult.Predictions[0].BoundingBox);
Assert.InRange(imageResult.Predictions[0].BoundingBox.Left, 0, 1);
Assert.InRange(imageResult.Predictions[0].BoundingBox.Top, 0, 1);
Assert.InRange(imageResult.Predictions[0].BoundingBox.Width, 0, 1);
Assert.InRange(imageResult.Predictions[0].BoundingBox.Height, 0, 1);
}
}
}
public async void ImageCounts()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
HttpMockServer.Initialize(this.GetType(), "ImageCounts", RecorderMode);
using (var project = CreateTrainedImageClassificationProject())
{
ICustomVisionTrainingClient client = BaseTests.GetTrainingClient();
var taggedImageCount = await client.GetTaggedImageCountAsync(project.ProjectId);
Assert.Equal(10, taggedImageCount);
var untaggedImageCount = await client.GetUntaggedImageCountAsync(project.ProjectId);
Assert.Equal(0, untaggedImageCount);
var iterationId = (await client.GetIterationsAsync(project.ProjectId))[0].Id;
var imagePerfCount = await client.GetImagePerformanceCountAsync(project.ProjectId, iterationId);
Assert.Equal(2, imagePerfCount);
}
}
}
public async void CreateImageFromUrl()
{
string imageUrl = "https://raw.githubusercontent.com/Microsoft/Cognitive-CustomVision-Windows/master/Samples/Images/Hemlock/hemlock_1.jpg";
using (MockContext context = MockContext.Start(this.GetType()))
{
HttpMockServer.Initialize(this.GetType(), "CreateImageFromUrl", RecorderMode);
using (ICustomVisionTrainingClient client = GetTrainingClient())
{
var newProject = client.CreateProjectAsync(projName, projDescription, ProjectBuilderHelper.FoodDomain).Result;
var tag = client.CreateTagAsync(newProject.Id, tagName).Result;
var urlImages = new ImageUrlCreateEntry[] { new ImageUrlCreateEntry(imageUrl) };
var tags = new Guid[] { tag.Id };
var imageCreatedFromUrl = client.CreateImagesFromUrlsAsync(newProject.Id, new ImageUrlCreateBatch(urlImages, tags)).Result;
Assert.True(imageCreatedFromUrl.IsBatchSuccessful);
Assert.Equal(1, imageCreatedFromUrl.Images.Count);
Assert.Equal(imageUrl, imageCreatedFromUrl.Images[0].SourceUrl);
Assert.Equal("OK", imageCreatedFromUrl.Images[0].Status);
Assert.NotEqual(Guid.Empty, imageCreatedFromUrl.Images[0].Image.Id);
Assert.NotEqual(0, imageCreatedFromUrl.Images[0].Image.Width);
Assert.NotEqual(0, imageCreatedFromUrl.Images[0].Image.Height);
Assert.NotEmpty(imageCreatedFromUrl.Images[0].Image.OriginalImageUri);
Assert.NotEmpty(imageCreatedFromUrl.Images[0].Image.ResizedImageUri);
Assert.NotEmpty(imageCreatedFromUrl.Images[0].Image.ThumbnailUri);
await client.DeleteProjectAsync(newProject.Id);
}
}
}
public async void QuerySuggestedImageCount()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
HttpMockServer.Initialize(this.GetType(), "QuerySuggestedImageCount", RecorderMode);
using (var project = CreateTrainedImageClassificationProject())
using (ICustomVisionTrainingClient client = BaseTests.GetTrainingClient())
{
// Add an untagged image we expect to be classified as tag1
var images = new ImageFileCreateEntry[] {
new ImageFileCreateEntry("suggest_ic.jpg", File.ReadAllBytes(Path.Combine("TestImages", "suggest_ic.jpg")))
};
var imageResult = await client.CreateImagesFromFilesAsync(project.ProjectId, new ImageFileCreateBatch(images));
Assert.True(imageResult.IsBatchSuccessful);
Assert.Equal(1, imageResult.Images.Count);
// Ask for suggestions
var suggestions = client.SuggestTagsAndRegions(project.ProjectId, project.IterationId, new Guid[] { imageResult.Images[0].Image.Id });
// Validate result
Assert.Equal(1, suggestions.Count);
var tags = await client.GetTagsAsync(project.ProjectId, project.IterationId);
// We expect to get Tag1 as the primary suggestion, so query with a high prediction
var countMapping = await client.QuerySuggestedImageCountAsync(project.ProjectId, project.IterationId, new TagFilter()
{
Threshold = 0.75,
TagIds = new Guid[] { tags[0].Id }
});
Assert.Equal(1, countMapping.Count);
Assert.Equal(1, countMapping[tags[0].Id.ToString()]);
// We expect to get Tag2 to have a low probabilty, make sure we don't find it with a high threshold
countMapping = await client.QuerySuggestedImageCountAsync(project.ProjectId, project.IterationId, new TagFilter()
{
Threshold = 0.75,
TagIds = new Guid[] { tags[1].Id }
});
Assert.Equal(1, countMapping.Count);
Assert.Equal(0, countMapping[tags[1].Id.ToString()]);
// Get results for all tags with a high threshold.
countMapping = await client.QuerySuggestedImageCountAsync(project.ProjectId, project.IterationId, new TagFilter()
{
Threshold = 0.75,
TagIds = tags.Select(t => t.Id).ToList()
});
Assert.Equal(2, countMapping.Count);
Assert.Equal(1, countMapping[tags[0].Id.ToString()]);
Assert.Equal(0, countMapping[tags[1].Id.ToString()]);
}
}
}
public static Iteration CreateTrainedImageClassificationProject(ICustomVisionTrainingClient client, string predictionResourceId, Guid?domain = null)
{
#if RECORD_MODE
var projName = ProjectPrefix + Guid.NewGuid().ToString();
// Create a project
var projDomain = domain.HasValue ? domain : GeneralDomain;
var project = client.CreateProject(projName, null, projDomain);
// Create two tags
var tagOne = client.CreateTag(project.Id, "Tag1");
var tagTwo = client.CreateTag(project.Id, "Tag2");
// Add five images for tag 1
var imagePath = Path.Combine("TestImages", "tag1");
var imageFileEntries = new List <ImageFileCreateEntry>();
foreach (var fileName in Directory.EnumerateFiles(imagePath))
{
imageFileEntries.Add(new ImageFileCreateEntry(fileName, File.ReadAllBytes(fileName)));
}
client.CreateImagesFromFiles(project.Id, new ImageFileCreateBatch(imageFileEntries, new List <Guid>(new Guid[] { tagOne.Id })));
// Add five images for tag 2
imagePath = Path.Combine("TestImages", "tag2");
imageFileEntries = new List <ImageFileCreateEntry>();
foreach (var fileName in Directory.EnumerateFiles(imagePath))
{
imageFileEntries.Add(new ImageFileCreateEntry(fileName, File.ReadAllBytes(fileName)));
}
client.CreateImagesFromFiles(project.Id, new ImageFileCreateBatch(imageFileEntries, new List <Guid>(new Guid[] { tagTwo.Id })));
// Train
var iteration = client.TrainProject(project.Id);
while (iteration.Status != "Completed")
{
Thread.Sleep(1000);
iteration = client.GetIteration(project.Id, iteration.Id);
}
Assert.True(client.PublishIteration(project.Id, iteration.Id, ClassificationPublishName, predictionResourceId));
// Get latest iteration
iteration = client.GetIteration(project.Id, iteration.Id);
// Make one prediction
string imageUrl = "https://raw.githubusercontent.com/Microsoft/Cognitive-CustomVision-Windows/master/Samples/Images/Test/test_image.jpg";
client.QuickTestImageUrl(project.Id, new ImageUrl(imageUrl), iteration.Id);
// Flush and re-init so we don't get all of the test setup in the session.
HttpMockServer.Flush();
HttpMockServer.Initialize(HttpMockServer.CallerIdentity, HttpMockServer.TestIdentity, HttpRecorderMode.Record);
return(iteration);
#else
return(null);
#endif
}
public async void RegionManipulation()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
HttpMockServer.Initialize(this.GetType(), "RegionManipulation", RecorderMode);
using (var project = CreateTrainedObjDetectionProject())
using (ICustomVisionTrainingClient client = BaseTests.GetTrainingClient())
{
var existingImage = (await client.GetTaggedImagesAsync(project.ProjectId)).First();
Assert.NotNull(existingImage);
var testTag = await client.CreateTagAsync(project.ProjectId, tagName);
var proposal = await client.GetImageRegionProposalsAsync(project.ProjectId, existingImage.Id);
Assert.True(proposal.Proposals.Count > 0);
Assert.Equal(project.ProjectId, proposal.ProjectId);
Assert.Equal(existingImage.Id, proposal.ImageId);
Assert.InRange(proposal.Proposals[0].Confidence, 0, 1);
var bbox = proposal.Proposals[0].BoundingBox;
List <ImageRegionCreateEntry> newRegions = new List <ImageRegionCreateEntry>();
newRegions.Add(new ImageRegionCreateEntry(existingImage.Id, testTag.Id, bbox.Left, bbox.Top, bbox.Width, bbox.Height));
var regions = await client.CreateImageRegionsAsync(project.ProjectId, new ImageRegionCreateBatch(newRegions));
Assert.Equal(1, regions.Created.Count);
Assert.Equal(0, regions.Duplicated.Count);
Assert.Equal(0, regions.Exceeded.Count);
Assert.Equal(bbox.Top, regions.Created[0].Top);
Assert.Equal(bbox.Left, regions.Created[0].Left);
Assert.Equal(bbox.Width, regions.Created[0].Width);
Assert.Equal(bbox.Height, regions.Created[0].Height);
Assert.Equal(existingImage.Id, regions.Created[0].ImageId);
Assert.Equal(testTag.Id, regions.Created[0].TagId);
Assert.NotEqual(Guid.Empty, regions.Created[0].RegionId);
var image = await client.GetImagesByIdsAsync(project.ProjectId, new List <Guid>(new Guid[] { existingImage.Id }));
Assert.Equal(1, image.Count);
Assert.Equal(2, image[0].Regions.Count);
await client.DeleteImageRegionsAsync(project.ProjectId, new List <Guid>(new Guid[] { regions.Created[0].RegionId }));
image = await client.GetImagesByIdsAsync(project.ProjectId, new List <Guid>(new Guid[] { existingImage.Id }));
Assert.Equal(1, image.Count);
Assert.Equal(1, image[0].Regions.Count);
await client.DeleteTagAsync(project.ProjectId, testTag.Id);
}
}
}
public static ICustomVisionTrainingClient GetTrainingClient(bool addHandler = true)
{
ICustomVisionTrainingClient client = (addHandler) ?
new CustomVisionTrainingClient(handlers: HttpMockServer.CreateInstance()) :
new CustomVisionTrainingClient();
client.ApiKey = TrainingKey;
client.Endpoint = Endpoint;
return(client);
}
public static ICustomVisionTrainingClient GetTrainingClient(bool addHandler = true)
{
var credentials = new ApiKeyServiceClientCredentials(TrainingKey);
ICustomVisionTrainingClient client = (addHandler) ?
new CustomVisionTrainingClient(credentials, handlers: HttpMockServer.CreateInstance()) :
new CustomVisionTrainingClient(credentials);
client.Endpoint = Endpoint;
return(client);
}
public static async Task <TrainingModels::Export> ExportIteration(
this ICustomVisionTrainingClient trainingApi, Guid projectId, Guid iterationId, string flavor = "onnx12", int timeoutInSecond = 30)
{
TimeSpan timeout = TimeSpan.FromSeconds(timeoutInSecond);
TrainingModels::Export exportIteration = null;
try
{
exportIteration = exportIteration = string.IsNullOrEmpty(flavor)
? await trainingApi.ExportIterationAsync(projectId, iterationId, platform)
: await trainingApi.ExportIterationAsync(projectId, iterationId, platform, flavor);
}
catch (HttpOperationException ex)
{
string exceptionContent = ex?.Response?.Content ?? string.Empty;
if (!exceptionContent.Contains("BadRequestExportAlreadyInProgress"))
{
throw ex;
}
}
DateTime startTime = DateTime.Now;
while (true)
{
IList <TrainingModels::Export> exports = await trainingApi.GetExportsAsync(projectId, iterationId);
exportIteration = string.IsNullOrEmpty(flavor)
? exports?.FirstOrDefault(x => string.Equals(x.Platform, platform, StringComparison.OrdinalIgnoreCase))
: exports?.FirstOrDefault(x => string.Equals(x.Platform, platform, StringComparison.OrdinalIgnoreCase) &&
string.Equals(x.Flavor, flavor, StringComparison.OrdinalIgnoreCase));
if (exportIteration?.Status == "Exporting")
{
await Task.Delay(1000);
}
else
{
break;
}
if (DateTime.Now - startTime > timeout)
{
throw new TimeoutException("The operation couldn't be completed due to a timeout.");
}
}
return(exportIteration);
}
public void Dispose()
{
#if RECORD_MODE
ICustomVisionTrainingClient client = BaseTests.GetTrainingClient(false);
foreach (var iter in client.GetIterations(this.ProjectId))
{
if (!string.IsNullOrEmpty(iter.PublishName))
{
client.UnpublishIteration(this.ProjectId, iter.Id);
}
}
client.DeleteProject(this.ProjectId);
#endif
}
public async void GetUntaggedImages()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
HttpMockServer.Initialize(this.GetType(), "GetUntaggedImages", RecorderMode);
using (var project = CreateTrainedImageClassificationProject())
{
ICustomVisionTrainingClient client = BaseTests.GetTrainingClient();
var images = await client.GetUntaggedImagesAsync(project.ProjectId);
Assert.Equal(0, images.Count);
}
}
}
public async void GetUntaggedImages()
{
using (MockContext context = MockContext.Start(this.GetType().Name))
{
HttpMockServer.Initialize(this.GetType().Name, "GetUntaggedImages", RecorderMode);
var projectId = CreateTrainedImageClassificationProject();
ICustomVisionTrainingClient client = GetTrainingClient();
var images = client.GetUntaggedImagesAsync(projectId).Result;
Assert.Equal(0, images.Count);
await client.DeleteProjectAsync(projectId);
}
}
public async void CreateDeleteProjectWithDomain()
{
using (MockContext context = MockContext.Start(this.GetType().Name))
{
HttpMockServer.Initialize(this.GetType().Name, "CreateDeleteProjectWithDomain", RecorderMode);
ICustomVisionTrainingClient client = GetTrainingClient();
var newProject = await client.CreateProjectAsync(projName, projDescription, FoodDomain);
Assert.Contains(projName, newProject.Name);
Assert.Equal(projDescription, newProject.Description);
Assert.Equal(FoodDomain, newProject.Settings.DomainId);
Assert.NotEqual(Guid.Empty, newProject.Id);
await client.DeleteProjectAsync(newProject.Id);
}
}
public static void CleanUpOldProjects(ICustomVisionTrainingClient client)
{
foreach (var project in client.GetProjects())
{
if (project.Name.StartsWith(ProjectPrefix))
{
System.Console.WriteLine("Cleaning old project: " + project.Name);
foreach (var iter in client.GetIterations(project.Id))
{
if (!string.IsNullOrEmpty(iter.PublishName))
{
client.UnpublishIteration(project.Id, iter.Id);
}
}
client.DeleteProject(project.Id);
}
}
}
public static async Task <TrainingModels::Export> ExportIteration(this ICustomVisionTrainingClient trainingApi, Guid projectId, Guid iterationId, int timeoutInSecond = 30)
{
TimeSpan timeout = TimeSpan.FromSeconds(timeoutInSecond);
TrainingModels::Export exportIteration = null;
try
{
exportIteration = await trainingApi.ExportIterationAsync(projectId, iterationId, platform);
}
catch (TrainingModels::CustomVisionErrorException ex)
{
if (ex.Body.Code != TrainingModels::CustomVisionErrorCodes.BadRequestExportAlreadyInProgress)
{
throw ex;
}
}
DateTime startTime = DateTime.Now;
while (true)
{
IList <TrainingModels::Export> exports = await trainingApi.GetExportsAsync(projectId, iterationId);
exportIteration = exports?.FirstOrDefault(x => string.Equals(x.Platform, platform, StringComparison.OrdinalIgnoreCase));
if (exportIteration?.Status == "Exporting")
{
await Task.Delay(1000);
}
else
{
break;
}
if (DateTime.Now - startTime > timeout)
{
throw new TimeoutException("The operation couldn't be completed due to a timeout.");
}
}
return(exportIteration);
}
public async void TrainAndPublishProject()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
HttpMockServer.Initialize(this.GetType(), "TrainAndPublishProject", RecorderMode);
using (var project = CreateTrainedImageClassificationProject())
{
ICustomVisionTrainingClient client = BaseTests.GetTrainingClient();
// Remove the last trained iteration so we can retrain
var iterationToDelete = client.GetIteration(project.ProjectId, project.IterationId);
var originalPublishName = iterationToDelete.PublishName;
await client.UnpublishIterationAsync(project.ProjectId, iterationToDelete.Id);
await client.DeleteIterationAsync(project.ProjectId, iterationToDelete.Id);
// Need to ensure we wait 1 second between training calls from the previous project.Or
// We get 429s.
Thread.Sleep(1000);
var trainedIteration = await client.TrainProjectAsync(project.ProjectId);
Assert.NotEqual(iterationToDelete.Name, trainedIteration.Name);
Assert.NotEqual(iterationToDelete.Id, trainedIteration.Id);
Assert.NotEqual(Guid.Empty, trainedIteration.Id);
Assert.True("Staging" == trainedIteration.Status || "Training" == trainedIteration.Status);
Assert.False(trainedIteration.Exportable);
Assert.Null(trainedIteration.PublishName);
// Wait for training to complete.
while (trainedIteration.Status != "Completed")
{
Thread.Sleep(1000);
trainedIteration = client.GetIteration(project.ProjectId, trainedIteration.Id);
}
// Verify we can republish using same name
Assert.True(client.PublishIteration(project.ProjectId, trainedIteration.Id, originalPublishName, BaseTests.PredictionResourceId));
project.IterationId = trainedIteration.Id;
}
}
}
public async void QuerySuggestedImages()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
HttpMockServer.Initialize(this.GetType(), "QuerySuggestedImages", RecorderMode);
using (var project = CreateTrainedImageClassificationProject())
using (ICustomVisionTrainingClient client = BaseTests.GetTrainingClient())
{
// Add an untagged image we expect to be classified as tag1
var images = new ImageFileCreateEntry[] {
new ImageFileCreateEntry("suggest_ic.jpg", File.ReadAllBytes(Path.Combine("TestImages", "suggest_ic.jpg")))
};
var imageResult = await client.CreateImagesFromFilesAsync(project.ProjectId, new ImageFileCreateBatch(images));
Assert.True(imageResult.IsBatchSuccessful);
Assert.Equal(1, imageResult.Images.Count);
// Ask for suggestions
var suggestions = client.SuggestTagsAndRegions(project.ProjectId, project.IterationId, new Guid[] { imageResult.Images[0].Image.Id });
// Validate result
Assert.Equal(1, suggestions.Count);
// Get tags and build a query
var tags = await client.GetTagsAsync(project.ProjectId, project.IterationId);
var query = new SuggestedTagAndRegionQueryToken()
{
MaxCount = 10,
TagIds = tags.Select(t => t.Id).ToList(),
Threshold = 0
};
// This will return all suggested images (1 in this case)
var suggestedImages = await client.QuerySuggestedImagesAsync(project.ProjectId, project.IterationId, query);
Assert.Equal(1, suggestedImages.Results.Count);
}
}
}
