private async void UpdateFolderLoadProgress(FolderLoadProgress progress)
{
this.FeedbackControl.Message.Content = progress.GetMessage();
this.FeedbackControl.ProgressBar.Value = progress.GetPercentage();
if (progress.DisplayBitmap)
{
// For good display quality the render size is ideally the markable canvas width. However, its width isn't known until layout of the
// FileViewPane completes, which occurs asynchronously on the UI thread from background worker thread execution. Therefore, start with
// a naive guess of the width and refine it as layout information becomes available. Profiling shows no difference in import speed
// for renders up to at least 1000 pixels wide or so, suggesting there's little reason to render at other than display size.
int markableCanvasWidth = (int)this.MarkableCanvas.Width;
if ((markableCanvasWidth > 0) && (markableCanvasWidth != progress.RenderWidthBestEstimate))
{
progress.RenderWidthBestEstimate = markableCanvasWidth;
}
BitmapSource bitmap;
if (progress.BitmapSource != null)
{
bitmap = progress.BitmapSource;
}
else
{
bitmap = await progress.CurrentFile.LoadBitmapAsync(this.FolderPath, progress.RenderWidthBestEstimate);
}
this.MarkableCanvas.SetNewImage(bitmap, null);
}
}
// out parameters can't be used in anonymous methods, so a separate pointer to backgroundWorker is required for return to the caller
private async Task<bool> TryBeginFolderLoadAsync(FolderLoad folderLoad)
{
List<FileInfo> filesToAdd = folderLoad.GetFiles();
if (filesToAdd.Count == 0)
{
// no images were found in folder; see if user wants to try again
MessageBox messageBox = new MessageBox("Select a folder containing images or videos?", this, MessageBoxButton.YesNo);
messageBox.Message.Problem = "There aren't any images or videos in the folder '" + this.FolderPath + "' so your image set is currentl empty.";
messageBox.Message.Reason = "\u2022 This folder has no images in it (files ending in .jpg)." + Environment.NewLine;
messageBox.Message.Reason += "\u2022 This folder has no videos in it (files ending in .avi or .mp4).";
messageBox.Message.Solution = "Choose Yes and select a folder containing images and/or videos or choose No and add files later via the File menu.";
messageBox.Message.Hint = "\u2022 The files may be in a subfolder of this folder." + Environment.NewLine;
messageBox.Message.Hint += "\u2022 If you need to set the image set's time zone before adding files choose No." + Environment.NewLine;
messageBox.Message.StatusImage = MessageBoxImage.Question;
if (messageBox.ShowDialog() == false)
{
return false;
}
IEnumerable<string> folderPaths;
if (this.ShowFolderSelectionDialog(out folderPaths))
{
folderLoad.FolderPaths.Clear();
folderLoad.FolderPaths.AddRange(folderPaths);
return await this.TryBeginFolderLoadAsync(folderLoad);
}
// exit if user changed their mind about trying again
return false;
}
// update UI for import (visibility is inverse of RunWorkerCompleted)
this.FeedbackControl.Visibility = Visibility.Visible;
this.FileNavigatorSlider.Visibility = Visibility.Collapsed;
IProgress<FolderLoadProgress> folderLoadStatus = new Progress<FolderLoadProgress>(this.UpdateFolderLoadProgress);
FolderLoadProgress folderLoadProgress = new FolderLoadProgress(filesToAdd.Count, this.MarkableCanvas.Width > 0 ? (int)this.MarkableCanvas.Width : (int)this.Width);
folderLoadStatus.Report(folderLoadProgress);
if (this.state.SkipDarkImagesCheck)
{
this.statusBar.SetMessage("Loading folders...");
}
else
{
this.statusBar.SetMessage("Loading folders (if this is slower than you like and dark image detection isn't needed you can select Skip dark check in the Options menu right now)...");
}
this.FileViewPane.IsActive = true;
// Load all files found
// First pass: Examine files to extract their basic properties and build a list of files not already in the database
//
// With dark calculations enabled:
// Profiling of a 1000 image load on quad core, single 80+MB/s capable SSD shows the following:
// - one thread: 100% normalized execution time, 35% CPU, 16MB/s disk (100% normalized time = 1 minute 58 seconds)
// - two threads: 55% normalized execution time, 50% CPU, 17MB/s disk (6.3% normalized time with dark checking skipped)
// - three threads: 46% normalized execution time, 70% CPU, 20MB/s disk
// This suggests memory bound operation due to image quality calculation. The overhead of displaying preview images is fairly low;
// normalized time is about 5% with both dark checking and previewing skipped.
//
// For now, try to get at least two threads as that captures most of the benefit from parallel operation. Video loading may be more CPU bound
// due to initial frame rendering and benefit from additional threads. This requires further investigation. It may also be desirable to reduce
// the pixel stride in image quality calculation, which would increase CPU load.
//
// With dark calculations disabled:
// The bottleneck's the SQL insert though using more than four threads (or possibly more threads than the number of physical processors as the
// test machine was quad core) results in slow progress on the first 20 files or so, making the optimum number of loading threads complex as it
// depends on amortizing startup lag across faster progress on the remaining import. As this is comparatively minor relative to SQL (at least
// for O(10,000) files for now just default to four threads in the disabled case.
//
// Note: the UI thread is free during loading. So if loading's going slow the user can switch off dark checking asynchronously to speed up
// loading.
//
// A sequential partitioner is used as this keeps the preview images displayed to the user in pretty much the same order as they're named,
// which is less confusing than TPL's default partitioning where the displayed image jumps back and forth through the image set. Pulling files
// nearly sequentially may also offer some minor disk performance benefit.
List<ImageRow> filesToInsert = new List<ImageRow>();
TimeZoneInfo imageSetTimeZone = this.dataHandler.FileDatabase.ImageSet.GetTimeZone();
await Task.Run(() => Parallel.ForEach(
new SequentialPartitioner<FileInfo>(filesToAdd),
Utilities.GetParallelOptions(this.state.SkipDarkImagesCheck ? Environment.ProcessorCount : 2),
(FileInfo fileInfo) =>
{
ImageRow file;
if (this.dataHandler.FileDatabase.GetOrCreateFile(fileInfo, imageSetTimeZone, out file))
{
// the database already has an entry for this file so skip it
// if needed, a separate list of files to update could be generated
return;
}
BitmapSource bitmapSource = null;
try
{
if (this.state.SkipDarkImagesCheck)
{
file.ImageQuality = FileSelection.Ok;
}
else
{
// load bitmap and determine its quality
// Parallel doesn't await async bodies so awaiting the load results in Parallel prematurely concluding the body task completed and
// dispatching another, resulting in system overload. The simplest solution is to block this worker thread, which is OK as there
// are many workers and they're decoupled from the UI thread.
bitmapSource = file.LoadBitmapAsync(this.FolderPath, folderLoadProgress.RenderWidthBestEstimate).GetAwaiter().GetResult();
if (bitmapSource == Constant.Images.CorruptFile.Value)
{
file.ImageQuality = FileSelection.Corrupt;
}
else
{
file.ImageQuality = bitmapSource.AsWriteable().IsDark(this.state.DarkPixelThreshold, this.state.DarkPixelRatioThreshold);
}
}
// see if the datetime can be updated from the metadata
file.TryReadDateTimeOriginalFromMetadata(this.FolderPath, imageSetTimeZone);
}
catch (Exception exception)
{
Debug.Fail(String.Format("Load of {0} failed as it's likely corrupted.", file.FileName), exception.ToString());
bitmapSource = Constant.Images.CorruptFile.Value;
file.ImageQuality = FileSelection.Corrupt;
}
lock (filesToInsert)
{
filesToInsert.Add(file);
}
DateTime utcNow = DateTime.UtcNow;
if (utcNow - folderLoadProgress.MostRecentStatusDispatch > this.state.Throttles.DesiredIntervalBetweenRenders)
{
lock (folderLoadProgress)
{
if (utcNow - folderLoadProgress.MostRecentStatusDispatch > this.state.Throttles.DesiredIntervalBetweenRenders)
{
// if file was already loaded for dark checking use the resulting bitmap
// otherwise, load the file for display
if (bitmapSource != null)
{
folderLoadProgress.BitmapSource = bitmapSource;
}
else
{
folderLoadProgress.BitmapSource = null;
}
folderLoadProgress.CurrentFile = file;
folderLoadProgress.CurrentFileIndex = filesToInsert.Count;
folderLoadProgress.DisplayBitmap = true;
folderLoadProgress.MostRecentStatusDispatch = utcNow;
folderLoadStatus.Report(folderLoadProgress);
}
}
}
}));
// Second pass: Update database
// Parallel execution above produces out of order results. Put them back in order so the user sees images in file name order when
// reviewing the image set.
folderLoadProgress.DatabaseInsert = true;
await Task.Run(() =>
{
filesToInsert = filesToInsert.OrderBy(file => Path.Combine(file.RelativePath, file.FileName)).ToList();
this.dataHandler.FileDatabase.AddFiles(filesToInsert, (ImageRow file, int fileIndex) =>
{
// skip reloading images to display as the user's already seen them import
folderLoadProgress.BitmapSource = null;
folderLoadProgress.CurrentFile = file;
folderLoadProgress.CurrentFileIndex = fileIndex;
folderLoadProgress.DisplayBitmap = false;
folderLoadStatus.Report(folderLoadProgress);
});
});
// hide the feedback bar, show the file slider
this.FeedbackControl.Visibility = Visibility.Collapsed;
this.FileNavigatorSlider.Visibility = Visibility.Visible;
await this.OnFolderLoadingCompleteAsync(true);
// tell the user how many files were loaded
this.MaybeShowFileCountsDialog(true);
return true;
}