private static async Task ProcessDirectoryAsync(string dir, bool forceUpdate = false)
{
Console.WriteLine($"Processing Directory {dir}");
var imageExtensions = new HashSet <string>(StringComparer.OrdinalIgnoreCase)
{
".png",
".jpg",
".bmp",
".jpeg",
".gif"
};
foreach (var file in
from file in Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories)
where imageExtensions.Contains(Path.GetExtension(file))
select file)
{
try
{
var fileName = Path.GetFileName(file);
// TODO: Find out if the image metadata has already been stored into DocumentDB.
ImageMetadata existing = null;
if (existing == null || forceUpdate)
{
Console.WriteLine($"Processing {file}");
// Resize (if needed) in order to reduce network latency and errors due to large files. Then store the result in a temporary file.
var resized = Util.ResizeIfRequired(file, 750);
Func <Task <Stream> > imageCB = async() => File.OpenRead(resized.Item2);
ImageInsights insights = null;
Console.Write("This is where we'd call Cognitive Services, ");
// TODO: Gather insights from Cognitive Services into ImageInsights
Console.Write("and store in Blob Storage, ");
// Store the image in Blob Storage
Console.Write("convert into ImageMetadata, ");
// Convert insights into ImageMetadata
Console.WriteLine("and write to DocumentDB.");
// Store that metadata into DocumentDB
// If it already exists, update the data. Otherwise, create a new document.
// Console.WriteLine($"Insights: {JsonConvert.SerializeObject(insights, Formatting.None)}");
}
else
{
Console.WriteLine($"Skipping {file}, exists and 'forceUpdate' not set.");
}
}
catch (Exception e)
{
Console.WriteLine($"Error: {e}");
}
}
}
/// <summary>
/// Store the ImageInsights into the metadata - pulls out tags and caption, stores number of faces and face details.
/// </summary>
/// <param name="insights"></param>
public void AddInsights(ImageInsights insights)
{
// Feel free to alter what you store here as you see fit.
// However, remember that alterations to the schema extend all the way through to the Azure Search Index,
// so you'll need to tune your Bot Framework code as well to ensure your Azure Search queries function
// as intended.
this.Caption = insights.VisionInsights?.Caption;
this.Tags = insights.VisionInsights?.Tags;
this.ArabicCaption = insights.ArabicVisionInsights?.Caption;
this.ArabicTags = insights.ArabicVisionInsights?.Tags;
this.NumFaces = insights.FaceInsights == null ? 0 : insights.FaceInsights.Length;
this.Faces = insights.FaceInsights;
}
/// <summary>
/// If we resize the image, we should resize the face rectangles in our insights appropriately.
/// </summary>
/// <param name="insights"></param>
/// <param name="resizeTransform"></param>
public static void AdjustFaceInsightsBasedOnResizing(ImageInsights insights, Tuple <double, double> resizeTransform)
{
if (resizeTransform == null)
{
return; // No resize was needed.
}
foreach (var faceInsight in insights.FaceInsights)
{
faceInsight.FaceRectangle.Left = (int)(faceInsight.FaceRectangle.Left * resizeTransform.Item1);
faceInsight.FaceRectangle.Top = (int)(faceInsight.FaceRectangle.Top * resizeTransform.Item2);
faceInsight.FaceRectangle.Width = (int)(faceInsight.FaceRectangle.Width * resizeTransform.Item1);
faceInsight.FaceRectangle.Height = (int)(faceInsight.FaceRectangle.Height * resizeTransform.Item2);
}
}
private async Task AddImageInsightsToViewModel(StorageFolder rootFolder, ImageInsights insights)
{
// Load image from file
BitmapImage bitmapImage = new BitmapImage();
await bitmapImage.SetSourceAsync((await(await rootFolder.GetFileAsync(insights.ImageId)).OpenStreamForReadAsync()).AsRandomAccessStream());
bitmapImage.DecodePixelHeight = 360;
// Create the view models
ImageInsightsViewModel insightsViewModel = new ImageInsightsViewModel(insights, bitmapImage);
this.AllResults.Add(insightsViewModel);
this.FilteredResults.Add(insightsViewModel);
foreach (var tag in insights.VisionInsights.Tags)
{
if (!this.TagFilters.Any(t => t.Tag == tag))
{
this.TagFilters.Add(new TagFilterViewModel(tag));
}
}
foreach (var faceInsights in insights.FaceInsights)
{
if (!this.FaceFilters.Any(f => f.FaceId == faceInsights.UniqueFaceId))
{
var imageStream = (await(await rootFolder.GetFileAsync(insights.ImageId)).OpenStreamForReadAsync()).AsRandomAccessStream();
ImageSource croppedFaced = await Util.GetCroppedBitmapAsync(imageStream, faceInsights.FaceRectangle);
this.FaceFilters.Add(new FaceFilterViewModel(faceInsights.UniqueFaceId, croppedFaced));
}
if (!this.EmotionFilters.Any(f => f.Emotion == faceInsights.TopEmotion))
{
this.EmotionFilters.Add(new EmotionFilterViewModel(faceInsights.TopEmotion));
}
}
}
private async Task ProcessImagesAsync(StorageFolder rootFolder)
{
this.progressRing.IsActive = true;
this.AllResults.Clear();
this.FilteredResults.Clear();
this.TagFilters.Clear();
this.EmotionFilters.Clear();
this.FaceFilters.Clear();
List <ImageInsights> insightsList = new List <ImageInsights>();
// see if we have pre-computed results and if so load it from the json file
try
{
StorageFile insightsResultFile = (await rootFolder.TryGetItemAsync("ImageInsights.json")) as StorageFile;
if (insightsResultFile != null)
{
using (StreamReader reader = new StreamReader(await insightsResultFile.OpenStreamForReadAsync()))
{
insightsList = JsonConvert.DeserializeObject <List <ImageInsights> >(await reader.ReadToEndAsync());
foreach (var insights in insightsList)
{
await AddImageInsightsToViewModel(rootFolder, insights);
}
}
}
}
catch
{
// We will just compute everything again in case of errors
}
if (!insightsList.Any())
{
// start with fresh face lists
await FaceListManager.ResetFaceLists();
// enumerate through the first 50 images and extract the insights
QueryOptions fileQueryOptions = new QueryOptions(CommonFileQuery.DefaultQuery, new[] { ".png", ".jpg", ".bmp", ".jpeg", ".gif" });
StorageFileQueryResult queryResult = rootFolder.CreateFileQueryWithOptions(fileQueryOptions);
foreach (var item in (await queryResult.GetFilesAsync(0, 50)))
{
try
{
// Resize (if needed) in order to reduce network latency and errors due to large files. Then store the result in a temporary file.
StorageFile resizedFile = await ApplicationData.Current.TemporaryFolder.CreateFileAsync("ImageCollectionInsights.jpg", CreationCollisionOption.GenerateUniqueName);
var resizeTransform = await Util.ResizePhoto(await item.OpenStreamForReadAsync(), 720, resizedFile);
// Send the file for processing
ImageInsights insights = await ImageProcessor.ProcessImageAsync(resizedFile.OpenStreamForReadAsync, item.Name);
// Delete resized file
await resizedFile.DeleteAsync();
// Adjust all FaceInsights coordinates based on the transform function between the original and resized photos
foreach (var faceInsight in insights.FaceInsights)
{
faceInsight.FaceRectangle.Left = (int)(faceInsight.FaceRectangle.Left * resizeTransform.Item1);
faceInsight.FaceRectangle.Top = (int)(faceInsight.FaceRectangle.Top * resizeTransform.Item2);
faceInsight.FaceRectangle.Width = (int)(faceInsight.FaceRectangle.Width * resizeTransform.Item1);
faceInsight.FaceRectangle.Height = (int)(faceInsight.FaceRectangle.Height * resizeTransform.Item2);
}
insightsList.Add(insights);
await AddImageInsightsToViewModel(rootFolder, insights);
}
catch (Exception ex)
{
await Util.GenericApiCallExceptionHandler(ex, "Error processing image.");
}
}
// save to json
StorageFile jsonFile = await rootFolder.CreateFileAsync("ImageInsights.json", CreationCollisionOption.ReplaceExisting);
using (StreamWriter writer = new StreamWriter(await jsonFile.OpenStreamForWriteAsync()))
{
string jsonStr = JsonConvert.SerializeObject(insightsList, Formatting.Indented);
await writer.WriteAsync(jsonStr);
}
}
var sortedTags = this.TagFilters.OrderBy(t => t.Tag).ToArray();
this.TagFilters.Clear();
this.TagFilters.AddRange(sortedTags);
var sortedEmotions = this.EmotionFilters.OrderBy(t => t.Emotion).ToArray();
this.EmotionFilters.Clear();
this.EmotionFilters.AddRange(sortedEmotions);
this.progressRing.IsActive = false;
}
/// <summary>
/// Store the ImageInsights into the metadata - pulls out tags and caption, stores number of faces and face details.
/// </summary>
/// <param name="insights"></param>
public void AddInsights(ImageInsights insights)
{
this.Caption = insights.Caption;
this.Tags = insights.Tags;
}
public ImageInsightsViewModel(ImageInsights insights, ImageSource imageSource)
{
this.Insights = insights;
this.ImageSource = imageSource;
}
private static async Task ProcessDirectoryAsync(string dir, bool forceUpdate = false)
{
Console.WriteLine($"Processing Directory {dir}");
var imageExtensions = new HashSet <string>(StringComparer.OrdinalIgnoreCase)
{
".png",
".jpg",
".bmp",
".jpeg",
".gif"
};
foreach (var file in
from file in Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories)
where imageExtensions.Contains(Path.GetExtension(file))
select file)
{
try
{
var fileName = Path.GetFileName(file);
var existing = await documentDb.FindDocumentByIdAsync <ImageMetadata>(fileName);
if (existing == null || forceUpdate)
{
Console.WriteLine($"Processing {file}");
// Resize (if needed) in order to reduce network latency and errors due to large files. Then store the result in a temporary file.
var resized = Util.ResizeIfRequired(file, 750);
Func <Task <Stream> > imageCB = async() => File.OpenRead(resized.Item2);
Console.Write("This is where we call Cognitive Services and store the results in ImageInsights, ");
ImageInsights insights = await ImageProcessor.ProcessImageAsync(imageCB, fileName);
Util.AdjustFaceInsightsBasedOnResizing(insights, resized.Item1);
Console.WriteLine($"Insights: {JsonConvert.SerializeObject(insights, Formatting.None)}");
Console.Write("store in Blob Storage, ");
var imageBlob = await blobStorage.UploadImageAsync(imageCB, fileName);
Console.Write("convert into ImageMetadata, ");
var metadata = new ImageMetadata(file);
metadata.AddInsights(insights);
metadata.BlobUri = imageBlob.Uri;
Console.WriteLine("and write to DocumentDB.");
if (existing == null)
{
metadata = (await documentDb.CreateDocumentIfNotExistsAsync(metadata, metadata.Id)).Item2;
}
else
{
metadata = await documentDb.UpdateDocumentAsync(metadata, metadata.Id);
}
}
else
{
Console.WriteLine($"Skipping {file}, exists and 'forceUpdate' not set.");
}
}
catch (Exception e)
{
Console.WriteLine($"Error: {e}");
}
}
}