// out parameters can't be used in anonymous methods, so a separate pointer to backgroundWorker is required for return to the caller
private async Task<bool> TryBeginFolderLoadAsync(FolderLoad folderLoad)
{
List<FileInfo> filesToAdd = folderLoad.GetFiles();
if (filesToAdd.Count == 0)
{
// no images were found in folder; see if user wants to try again
MessageBox messageBox = new MessageBox("Select a folder containing images or videos?", this, MessageBoxButton.YesNo);
messageBox.Message.Problem = "There aren't any images or videos in the folder '" + this.FolderPath + "' so your image set is currentl empty.";
messageBox.Message.Reason = "\u2022 This folder has no images in it (files ending in .jpg)." + Environment.NewLine;
messageBox.Message.Reason += "\u2022 This folder has no videos in it (files ending in .avi or .mp4).";
messageBox.Message.Solution = "Choose Yes and select a folder containing images and/or videos or choose No and add files later via the File menu.";
messageBox.Message.Hint = "\u2022 The files may be in a subfolder of this folder." + Environment.NewLine;
messageBox.Message.Hint += "\u2022 If you need to set the image set's time zone before adding files choose No." + Environment.NewLine;
messageBox.Message.StatusImage = MessageBoxImage.Question;
if (messageBox.ShowDialog() == false)
{
return false;
}
IEnumerable<string> folderPaths;
if (this.ShowFolderSelectionDialog(out folderPaths))
{
folderLoad.FolderPaths.Clear();
folderLoad.FolderPaths.AddRange(folderPaths);
return await this.TryBeginFolderLoadAsync(folderLoad);
}
// exit if user changed their mind about trying again
return false;
}
// update UI for import (visibility is inverse of RunWorkerCompleted)
this.FeedbackControl.Visibility = Visibility.Visible;
this.FileNavigatorSlider.Visibility = Visibility.Collapsed;
IProgress<FolderLoadProgress> folderLoadStatus = new Progress<FolderLoadProgress>(this.UpdateFolderLoadProgress);
FolderLoadProgress folderLoadProgress = new FolderLoadProgress(filesToAdd.Count, this.MarkableCanvas.Width > 0 ? (int)this.MarkableCanvas.Width : (int)this.Width);
folderLoadStatus.Report(folderLoadProgress);
if (this.state.SkipDarkImagesCheck)
{
this.statusBar.SetMessage("Loading folders...");
}
else
{
this.statusBar.SetMessage("Loading folders (if this is slower than you like and dark image detection isn't needed you can select Skip dark check in the Options menu right now)...");
}
this.FileViewPane.IsActive = true;
// Load all files found
// First pass: Examine files to extract their basic properties and build a list of files not already in the database
//
// With dark calculations enabled:
// Profiling of a 1000 image load on quad core, single 80+MB/s capable SSD shows the following:
// - one thread: 100% normalized execution time, 35% CPU, 16MB/s disk (100% normalized time = 1 minute 58 seconds)
// - two threads: 55% normalized execution time, 50% CPU, 17MB/s disk (6.3% normalized time with dark checking skipped)
// - three threads: 46% normalized execution time, 70% CPU, 20MB/s disk
// This suggests memory bound operation due to image quality calculation. The overhead of displaying preview images is fairly low;
// normalized time is about 5% with both dark checking and previewing skipped.
//
// For now, try to get at least two threads as that captures most of the benefit from parallel operation. Video loading may be more CPU bound
// due to initial frame rendering and benefit from additional threads. This requires further investigation. It may also be desirable to reduce
// the pixel stride in image quality calculation, which would increase CPU load.
//
// With dark calculations disabled:
// The bottleneck's the SQL insert though using more than four threads (or possibly more threads than the number of physical processors as the
// test machine was quad core) results in slow progress on the first 20 files or so, making the optimum number of loading threads complex as it
// depends on amortizing startup lag across faster progress on the remaining import. As this is comparatively minor relative to SQL (at least
// for O(10,000) files for now just default to four threads in the disabled case.
//
// Note: the UI thread is free during loading. So if loading's going slow the user can switch off dark checking asynchronously to speed up
// loading.
//
// A sequential partitioner is used as this keeps the preview images displayed to the user in pretty much the same order as they're named,
// which is less confusing than TPL's default partitioning where the displayed image jumps back and forth through the image set. Pulling files
// nearly sequentially may also offer some minor disk performance benefit.
List<ImageRow> filesToInsert = new List<ImageRow>();
TimeZoneInfo imageSetTimeZone = this.dataHandler.FileDatabase.ImageSet.GetTimeZone();
await Task.Run(() => Parallel.ForEach(
new SequentialPartitioner<FileInfo>(filesToAdd),
Utilities.GetParallelOptions(this.state.SkipDarkImagesCheck ? Environment.ProcessorCount : 2),
(FileInfo fileInfo) =>
{
ImageRow file;
if (this.dataHandler.FileDatabase.GetOrCreateFile(fileInfo, imageSetTimeZone, out file))
{
// the database already has an entry for this file so skip it
// if needed, a separate list of files to update could be generated
return;
}
BitmapSource bitmapSource = null;
try
{
if (this.state.SkipDarkImagesCheck)
{
file.ImageQuality = FileSelection.Ok;
}
else
{
// load bitmap and determine its quality
// Parallel doesn't await async bodies so awaiting the load results in Parallel prematurely concluding the body task completed and
// dispatching another, resulting in system overload. The simplest solution is to block this worker thread, which is OK as there
// are many workers and they're decoupled from the UI thread.
bitmapSource = file.LoadBitmapAsync(this.FolderPath, folderLoadProgress.RenderWidthBestEstimate).GetAwaiter().GetResult();
if (bitmapSource == Constant.Images.CorruptFile.Value)
{
file.ImageQuality = FileSelection.Corrupt;
}
else
{
file.ImageQuality = bitmapSource.AsWriteable().IsDark(this.state.DarkPixelThreshold, this.state.DarkPixelRatioThreshold);
}
}
// see if the datetime can be updated from the metadata
file.TryReadDateTimeOriginalFromMetadata(this.FolderPath, imageSetTimeZone);
}
catch (Exception exception)
{
Debug.Fail(String.Format("Load of {0} failed as it's likely corrupted.", file.FileName), exception.ToString());
bitmapSource = Constant.Images.CorruptFile.Value;
file.ImageQuality = FileSelection.Corrupt;
}
lock (filesToInsert)
{
filesToInsert.Add(file);
}
DateTime utcNow = DateTime.UtcNow;
if (utcNow - folderLoadProgress.MostRecentStatusDispatch > this.state.Throttles.DesiredIntervalBetweenRenders)
{
lock (folderLoadProgress)
{
if (utcNow - folderLoadProgress.MostRecentStatusDispatch > this.state.Throttles.DesiredIntervalBetweenRenders)
{
// if file was already loaded for dark checking use the resulting bitmap
// otherwise, load the file for display
if (bitmapSource != null)
{
folderLoadProgress.BitmapSource = bitmapSource;
}
else
{
folderLoadProgress.BitmapSource = null;
}
folderLoadProgress.CurrentFile = file;
folderLoadProgress.CurrentFileIndex = filesToInsert.Count;
folderLoadProgress.DisplayBitmap = true;
folderLoadProgress.MostRecentStatusDispatch = utcNow;
folderLoadStatus.Report(folderLoadProgress);
}
}
}
}));
// Second pass: Update database
// Parallel execution above produces out of order results. Put them back in order so the user sees images in file name order when
// reviewing the image set.
folderLoadProgress.DatabaseInsert = true;
await Task.Run(() =>
{
filesToInsert = filesToInsert.OrderBy(file => Path.Combine(file.RelativePath, file.FileName)).ToList();
this.dataHandler.FileDatabase.AddFiles(filesToInsert, (ImageRow file, int fileIndex) =>
{
// skip reloading images to display as the user's already seen them import
folderLoadProgress.BitmapSource = null;
folderLoadProgress.CurrentFile = file;
folderLoadProgress.CurrentFileIndex = fileIndex;
folderLoadProgress.DisplayBitmap = false;
folderLoadStatus.Report(folderLoadProgress);
});
});
// hide the feedback bar, show the file slider
this.FeedbackControl.Visibility = Visibility.Collapsed;
this.FileNavigatorSlider.Visibility = Visibility.Visible;
await this.OnFolderLoadingCompleteAsync(true);
// tell the user how many files were loaded
this.MaybeShowFileCountsDialog(true);
return true;
}
/// <summary>
/// Load the specified database template and then the associated file database.
/// </summary>
/// <param name="templateDatabasePath">Fully qualified path to the template database file.</param>
/// <returns>true only if both the template and image database file are loaded (regardless of whether any images were loaded), false otherwise</returns>
/// <remarks>This method doesn't particularly need to be public. But making it private imposes substantial complexity in invoking it via PrivateObject
/// in unit tests.</remarks>
public async Task<bool> TryOpenTemplateAndBeginLoadFoldersAsync(string templateDatabasePath)
{
// Try to create or open the template database
TemplateDatabase templateDatabase;
if (TemplateDatabase.TryCreateOrOpen(templateDatabasePath, out templateDatabase) == false)
{
// notify the user the template couldn't be loaded
MessageBox messageBox = new MessageBox("Carnassial could not load the template.", this);
messageBox.Message.Problem = "Carnassial could not load " + Path.GetFileName(templateDatabasePath) + Environment.NewLine;
messageBox.Message.Reason = "\u2022 The template was created with the Timelapse template editor instead of the Carnassial editor." + Environment.NewLine;
messageBox.Message.Reason = "\u2022 The template may be corrupted or somehow otherwise invalid.";
messageBox.Message.Solution = String.Format("You may have to recreate the template, restore it from the {0} folder, or use another copy of it if you have one.", Constant.File.BackupFolder);
messageBox.Message.Result = "Carnassial won't do anything. You can try to select another template file.";
messageBox.Message.Hint = "If the template can't be opened in a SQLite database editor the file is corrupt.";
messageBox.Message.StatusImage = MessageBoxImage.Error;
messageBox.ShowDialog();
this.state.MostRecentImageSets.TryRemove(templateDatabasePath);
this.MenuFileRecentImageSets_Refresh();
return false;
}
// Try to get the file database file path
// importImages will be true if it's a new image database file (meaning the user will be prompted import some images)
string fileDatabaseFilePath;
bool addFiles;
if (this.TrySelectDatabaseFile(templateDatabasePath, out fileDatabaseFilePath, out addFiles) == false)
{
// No image database file was selected
templateDatabase.Dispose();
return false;
}
// Before running from an existing file database, check the controls in the template database are compatible with those
// of the file database.
FileDatabase fileDatabase;
if (FileDatabase.TryCreateOrOpen(fileDatabaseFilePath, templateDatabase, this.state.OrderFilesByDateTime, this.state.CustomSelectionTermCombiningOperator, out fileDatabase) == false)
{
// notify the user the database couldn't be loaded
MessageBox messageBox = new MessageBox("Carnassial could not load the database.", this);
messageBox.Message.Problem = "Carnassial could not load " + Path.GetFileName(fileDatabaseFilePath) + Environment.NewLine;
messageBox.Message.Reason = "\u2022 The database was created with Timelapse instead of Carnassial." + Environment.NewLine;
messageBox.Message.Reason = "\u2022 The database may be corrupted or somehow otherwise invalid.";
messageBox.Message.Solution = String.Format("You may have to recreate the database, restore it from the {0} folder, or use another copy of it if you have one.", Constant.File.BackupFolder);
messageBox.Message.Result = "Carnassial won't do anything. You can try to select another template or database file.";
messageBox.Message.Hint = "If the database can't be opened in a SQLite database editor the file is corrupt.";
messageBox.Message.StatusImage = MessageBoxImage.Error;
messageBox.ShowDialog();
return false;
}
templateDatabase.Dispose();
if (fileDatabase.ControlSynchronizationIssues.Count > 0)
{
TemplateSynchronization templatesNotCompatibleDialog = new TemplateSynchronization(fileDatabase.ControlSynchronizationIssues, this);
bool? result = templatesNotCompatibleDialog.ShowDialog();
if (result == true)
{
// user indicated not to update to the current template so exit.
Application.Current.Shutdown();
return false;
}
// user indicated to run with the stale copy of the template found in the image database
}
// valid template and file database loaded
// generate and render the data entry controls regardless of whether there are actually any files in the file database.
this.dataHandler = new DataEntryHandler(fileDatabase);
this.DataEntryControls.CreateControls(fileDatabase, this.dataHandler);
this.SetUserInterfaceCallbacks();
this.MenuFileRecentImageSets_Refresh();
this.state.MostRecentFileAddFolderPath = fileDatabase.FolderPath;
this.state.MostRecentImageSets.SetMostRecent(templateDatabasePath);
this.Title = Path.GetFileName(fileDatabase.FilePath) + " - " + Constant.MainWindowBaseTitle;
// If this is a new file database, try to load files (if any) from the folder...
if (addFiles)
{
FolderLoad folderLoad = new FolderLoad();
folderLoad.FolderPaths.Add(this.FolderPath);
await this.TryBeginFolderLoadAsync(folderLoad);
}
await this.OnFolderLoadingCompleteAsync(false);
return true;
}
private async void MenuFileAddFilesToImageSet_Click(object sender, RoutedEventArgs e)
{
IEnumerable<string> folderPaths;
if (this.ShowFolderSelectionDialog(out folderPaths))
{
FolderLoad folderLoad = new FolderLoad();
folderLoad.FolderPaths.AddRange(folderPaths);
await this.TryBeginFolderLoadAsync(folderLoad);
}
}