private static async Task ProcessDirectoryAsync(string dir, bool forceUpdate = false) { Console.WriteLine($"Processing Directory {dir}"); var imageExtensions = new HashSet <string>(StringComparer.OrdinalIgnoreCase) { ".png", ".jpg", ".bmp", ".jpeg", ".gif" }; foreach (var file in from file in Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories) where imageExtensions.Contains(Path.GetExtension(file)) select file) { try { var fileName = Path.GetFileName(file); // TODO: Find out if the image metadata has already been stored into DocumentDB. ImageMetadata existing = null; if (existing == null || forceUpdate) { Console.WriteLine($"Processing {file}"); // Resize (if needed) in order to reduce network latency and errors due to large files. Then store the result in a temporary file. var resized = Util.ResizeIfRequired(file, 750); Func <Task <Stream> > imageCB = async() => File.OpenRead(resized.Item2); ImageInsights insights = null; Console.Write("This is where we'd call Cognitive Services, "); // TODO: Gather insights from Cognitive Services into ImageInsights Console.Write("and store in Blob Storage, "); // Store the image in Blob Storage Console.Write("convert into ImageMetadata, "); // Convert insights into ImageMetadata Console.WriteLine("and write to DocumentDB."); // Store that metadata into DocumentDB // If it already exists, update the data. Otherwise, create a new document. // Console.WriteLine($"Insights: {JsonConvert.SerializeObject(insights, Formatting.None)}"); } else { Console.WriteLine($"Skipping {file}, exists and 'forceUpdate' not set."); } } catch (Exception e) { Console.WriteLine($"Error: {e}"); } } }
/// <summary> /// Store the ImageInsights into the metadata - pulls out tags and caption, stores number of faces and face details. /// </summary> /// <param name="insights"></param> public void AddInsights(ImageInsights insights) { // Feel free to alter what you store here as you see fit. // However, remember that alterations to the schema extend all the way through to the Azure Search Index, // so you'll need to tune your Bot Framework code as well to ensure your Azure Search queries function // as intended. this.Caption = insights.VisionInsights?.Caption; this.Tags = insights.VisionInsights?.Tags; this.ArabicCaption = insights.ArabicVisionInsights?.Caption; this.ArabicTags = insights.ArabicVisionInsights?.Tags; this.NumFaces = insights.FaceInsights == null ? 0 : insights.FaceInsights.Length; this.Faces = insights.FaceInsights; }
/// <summary> /// If we resize the image, we should resize the face rectangles in our insights appropriately. /// </summary> /// <param name="insights"></param> /// <param name="resizeTransform"></param> public static void AdjustFaceInsightsBasedOnResizing(ImageInsights insights, Tuple <double, double> resizeTransform) { if (resizeTransform == null) { return; // No resize was needed. } foreach (var faceInsight in insights.FaceInsights) { faceInsight.FaceRectangle.Left = (int)(faceInsight.FaceRectangle.Left * resizeTransform.Item1); faceInsight.FaceRectangle.Top = (int)(faceInsight.FaceRectangle.Top * resizeTransform.Item2); faceInsight.FaceRectangle.Width = (int)(faceInsight.FaceRectangle.Width * resizeTransform.Item1); faceInsight.FaceRectangle.Height = (int)(faceInsight.FaceRectangle.Height * resizeTransform.Item2); } }
private async Task AddImageInsightsToViewModel(StorageFolder rootFolder, ImageInsights insights) { // Load image from file BitmapImage bitmapImage = new BitmapImage(); await bitmapImage.SetSourceAsync((await(await rootFolder.GetFileAsync(insights.ImageId)).OpenStreamForReadAsync()).AsRandomAccessStream()); bitmapImage.DecodePixelHeight = 360; // Create the view models ImageInsightsViewModel insightsViewModel = new ImageInsightsViewModel(insights, bitmapImage); this.AllResults.Add(insightsViewModel); this.FilteredResults.Add(insightsViewModel); foreach (var tag in insights.VisionInsights.Tags) { if (!this.TagFilters.Any(t => t.Tag == tag)) { this.TagFilters.Add(new TagFilterViewModel(tag)); } } foreach (var faceInsights in insights.FaceInsights) { if (!this.FaceFilters.Any(f => f.FaceId == faceInsights.UniqueFaceId)) { var imageStream = (await(await rootFolder.GetFileAsync(insights.ImageId)).OpenStreamForReadAsync()).AsRandomAccessStream(); ImageSource croppedFaced = await Util.GetCroppedBitmapAsync(imageStream, faceInsights.FaceRectangle); this.FaceFilters.Add(new FaceFilterViewModel(faceInsights.UniqueFaceId, croppedFaced)); } if (!this.EmotionFilters.Any(f => f.Emotion == faceInsights.TopEmotion)) { this.EmotionFilters.Add(new EmotionFilterViewModel(faceInsights.TopEmotion)); } } }
private async Task ProcessImagesAsync(StorageFolder rootFolder) { this.progressRing.IsActive = true; this.AllResults.Clear(); this.FilteredResults.Clear(); this.TagFilters.Clear(); this.EmotionFilters.Clear(); this.FaceFilters.Clear(); List <ImageInsights> insightsList = new List <ImageInsights>(); // see if we have pre-computed results and if so load it from the json file try { StorageFile insightsResultFile = (await rootFolder.TryGetItemAsync("ImageInsights.json")) as StorageFile; if (insightsResultFile != null) { using (StreamReader reader = new StreamReader(await insightsResultFile.OpenStreamForReadAsync())) { insightsList = JsonConvert.DeserializeObject <List <ImageInsights> >(await reader.ReadToEndAsync()); foreach (var insights in insightsList) { await AddImageInsightsToViewModel(rootFolder, insights); } } } } catch { // We will just compute everything again in case of errors } if (!insightsList.Any()) { // start with fresh face lists await FaceListManager.ResetFaceLists(); // enumerate through the first 50 images and extract the insights QueryOptions fileQueryOptions = new QueryOptions(CommonFileQuery.DefaultQuery, new[] { ".png", ".jpg", ".bmp", ".jpeg", ".gif" }); StorageFileQueryResult queryResult = rootFolder.CreateFileQueryWithOptions(fileQueryOptions); foreach (var item in (await queryResult.GetFilesAsync(0, 50))) { try { // Resize (if needed) in order to reduce network latency and errors due to large files. Then store the result in a temporary file. StorageFile resizedFile = await ApplicationData.Current.TemporaryFolder.CreateFileAsync("ImageCollectionInsights.jpg", CreationCollisionOption.GenerateUniqueName); var resizeTransform = await Util.ResizePhoto(await item.OpenStreamForReadAsync(), 720, resizedFile); // Send the file for processing ImageInsights insights = await ImageProcessor.ProcessImageAsync(resizedFile.OpenStreamForReadAsync, item.Name); // Delete resized file await resizedFile.DeleteAsync(); // Adjust all FaceInsights coordinates based on the transform function between the original and resized photos foreach (var faceInsight in insights.FaceInsights) { faceInsight.FaceRectangle.Left = (int)(faceInsight.FaceRectangle.Left * resizeTransform.Item1); faceInsight.FaceRectangle.Top = (int)(faceInsight.FaceRectangle.Top * resizeTransform.Item2); faceInsight.FaceRectangle.Width = (int)(faceInsight.FaceRectangle.Width * resizeTransform.Item1); faceInsight.FaceRectangle.Height = (int)(faceInsight.FaceRectangle.Height * resizeTransform.Item2); } insightsList.Add(insights); await AddImageInsightsToViewModel(rootFolder, insights); } catch (Exception ex) { await Util.GenericApiCallExceptionHandler(ex, "Error processing image."); } } // save to json StorageFile jsonFile = await rootFolder.CreateFileAsync("ImageInsights.json", CreationCollisionOption.ReplaceExisting); using (StreamWriter writer = new StreamWriter(await jsonFile.OpenStreamForWriteAsync())) { string jsonStr = JsonConvert.SerializeObject(insightsList, Formatting.Indented); await writer.WriteAsync(jsonStr); } } var sortedTags = this.TagFilters.OrderBy(t => t.Tag).ToArray(); this.TagFilters.Clear(); this.TagFilters.AddRange(sortedTags); var sortedEmotions = this.EmotionFilters.OrderBy(t => t.Emotion).ToArray(); this.EmotionFilters.Clear(); this.EmotionFilters.AddRange(sortedEmotions); this.progressRing.IsActive = false; }
/// <summary> /// Store the ImageInsights into the metadata - pulls out tags and caption, stores number of faces and face details. /// </summary> /// <param name="insights"></param> public void AddInsights(ImageInsights insights) { this.Caption = insights.Caption; this.Tags = insights.Tags; }
public ImageInsightsViewModel(ImageInsights insights, ImageSource imageSource) { this.Insights = insights; this.ImageSource = imageSource; }
private static async Task ProcessDirectoryAsync(string dir, bool forceUpdate = false) { Console.WriteLine($"Processing Directory {dir}"); var imageExtensions = new HashSet <string>(StringComparer.OrdinalIgnoreCase) { ".png", ".jpg", ".bmp", ".jpeg", ".gif" }; foreach (var file in from file in Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories) where imageExtensions.Contains(Path.GetExtension(file)) select file) { try { var fileName = Path.GetFileName(file); var existing = await documentDb.FindDocumentByIdAsync <ImageMetadata>(fileName); if (existing == null || forceUpdate) { Console.WriteLine($"Processing {file}"); // Resize (if needed) in order to reduce network latency and errors due to large files. Then store the result in a temporary file. var resized = Util.ResizeIfRequired(file, 750); Func <Task <Stream> > imageCB = async() => File.OpenRead(resized.Item2); Console.Write("This is where we call Cognitive Services and store the results in ImageInsights, "); ImageInsights insights = await ImageProcessor.ProcessImageAsync(imageCB, fileName); Util.AdjustFaceInsightsBasedOnResizing(insights, resized.Item1); Console.WriteLine($"Insights: {JsonConvert.SerializeObject(insights, Formatting.None)}"); Console.Write("store in Blob Storage, "); var imageBlob = await blobStorage.UploadImageAsync(imageCB, fileName); Console.Write("convert into ImageMetadata, "); var metadata = new ImageMetadata(file); metadata.AddInsights(insights); metadata.BlobUri = imageBlob.Uri; Console.WriteLine("and write to DocumentDB."); if (existing == null) { metadata = (await documentDb.CreateDocumentIfNotExistsAsync(metadata, metadata.Id)).Item2; } else { metadata = await documentDb.UpdateDocumentAsync(metadata, metadata.Id); } } else { Console.WriteLine($"Skipping {file}, exists and 'forceUpdate' not set."); } } catch (Exception e) { Console.WriteLine($"Error: {e}"); } } }