private async void UpdateFolderLoadProgress(FolderLoadProgress progress) { this.FeedbackControl.Message.Content = progress.GetMessage(); this.FeedbackControl.ProgressBar.Value = progress.GetPercentage(); if (progress.DisplayBitmap) { // For good display quality the render size is ideally the markable canvas width. However, its width isn't known until layout of the // FileViewPane completes, which occurs asynchronously on the UI thread from background worker thread execution. Therefore, start with // a naive guess of the width and refine it as layout information becomes available. Profiling shows no difference in import speed // for renders up to at least 1000 pixels wide or so, suggesting there's little reason to render at other than display size. int markableCanvasWidth = (int)this.MarkableCanvas.Width; if ((markableCanvasWidth > 0) && (markableCanvasWidth != progress.RenderWidthBestEstimate)) { progress.RenderWidthBestEstimate = markableCanvasWidth; } BitmapSource bitmap; if (progress.BitmapSource != null) { bitmap = progress.BitmapSource; } else { bitmap = await progress.CurrentFile.LoadBitmapAsync(this.FolderPath, progress.RenderWidthBestEstimate); } this.MarkableCanvas.SetNewImage(bitmap, null); } }
// out parameters can't be used in anonymous methods, so a separate pointer to backgroundWorker is required for return to the caller private async Task<bool> TryBeginFolderLoadAsync(FolderLoad folderLoad) { List<FileInfo> filesToAdd = folderLoad.GetFiles(); if (filesToAdd.Count == 0) { // no images were found in folder; see if user wants to try again MessageBox messageBox = new MessageBox("Select a folder containing images or videos?", this, MessageBoxButton.YesNo); messageBox.Message.Problem = "There aren't any images or videos in the folder '" + this.FolderPath + "' so your image set is currentl empty."; messageBox.Message.Reason = "\u2022 This folder has no images in it (files ending in .jpg)." + Environment.NewLine; messageBox.Message.Reason += "\u2022 This folder has no videos in it (files ending in .avi or .mp4)."; messageBox.Message.Solution = "Choose Yes and select a folder containing images and/or videos or choose No and add files later via the File menu."; messageBox.Message.Hint = "\u2022 The files may be in a subfolder of this folder." + Environment.NewLine; messageBox.Message.Hint += "\u2022 If you need to set the image set's time zone before adding files choose No." + Environment.NewLine; messageBox.Message.StatusImage = MessageBoxImage.Question; if (messageBox.ShowDialog() == false) { return false; } IEnumerable<string> folderPaths; if (this.ShowFolderSelectionDialog(out folderPaths)) { folderLoad.FolderPaths.Clear(); folderLoad.FolderPaths.AddRange(folderPaths); return await this.TryBeginFolderLoadAsync(folderLoad); } // exit if user changed their mind about trying again return false; } // update UI for import (visibility is inverse of RunWorkerCompleted) this.FeedbackControl.Visibility = Visibility.Visible; this.FileNavigatorSlider.Visibility = Visibility.Collapsed; IProgress<FolderLoadProgress> folderLoadStatus = new Progress<FolderLoadProgress>(this.UpdateFolderLoadProgress); FolderLoadProgress folderLoadProgress = new FolderLoadProgress(filesToAdd.Count, this.MarkableCanvas.Width > 0 ? (int)this.MarkableCanvas.Width : (int)this.Width); folderLoadStatus.Report(folderLoadProgress); if (this.state.SkipDarkImagesCheck) { this.statusBar.SetMessage("Loading folders..."); } else { this.statusBar.SetMessage("Loading folders (if this is slower than you like and dark image detection isn't needed you can select Skip dark check in the Options menu right now)..."); } this.FileViewPane.IsActive = true; // Load all files found // First pass: Examine files to extract their basic properties and build a list of files not already in the database // // With dark calculations enabled: // Profiling of a 1000 image load on quad core, single 80+MB/s capable SSD shows the following: // - one thread: 100% normalized execution time, 35% CPU, 16MB/s disk (100% normalized time = 1 minute 58 seconds) // - two threads: 55% normalized execution time, 50% CPU, 17MB/s disk (6.3% normalized time with dark checking skipped) // - three threads: 46% normalized execution time, 70% CPU, 20MB/s disk // This suggests memory bound operation due to image quality calculation. The overhead of displaying preview images is fairly low; // normalized time is about 5% with both dark checking and previewing skipped. // // For now, try to get at least two threads as that captures most of the benefit from parallel operation. Video loading may be more CPU bound // due to initial frame rendering and benefit from additional threads. This requires further investigation. It may also be desirable to reduce // the pixel stride in image quality calculation, which would increase CPU load. // // With dark calculations disabled: // The bottleneck's the SQL insert though using more than four threads (or possibly more threads than the number of physical processors as the // test machine was quad core) results in slow progress on the first 20 files or so, making the optimum number of loading threads complex as it // depends on amortizing startup lag across faster progress on the remaining import. As this is comparatively minor relative to SQL (at least // for O(10,000) files for now just default to four threads in the disabled case. // // Note: the UI thread is free during loading. So if loading's going slow the user can switch off dark checking asynchronously to speed up // loading. // // A sequential partitioner is used as this keeps the preview images displayed to the user in pretty much the same order as they're named, // which is less confusing than TPL's default partitioning where the displayed image jumps back and forth through the image set. Pulling files // nearly sequentially may also offer some minor disk performance benefit. List<ImageRow> filesToInsert = new List<ImageRow>(); TimeZoneInfo imageSetTimeZone = this.dataHandler.FileDatabase.ImageSet.GetTimeZone(); await Task.Run(() => Parallel.ForEach( new SequentialPartitioner<FileInfo>(filesToAdd), Utilities.GetParallelOptions(this.state.SkipDarkImagesCheck ? Environment.ProcessorCount : 2), (FileInfo fileInfo) => { ImageRow file; if (this.dataHandler.FileDatabase.GetOrCreateFile(fileInfo, imageSetTimeZone, out file)) { // the database already has an entry for this file so skip it // if needed, a separate list of files to update could be generated return; } BitmapSource bitmapSource = null; try { if (this.state.SkipDarkImagesCheck) { file.ImageQuality = FileSelection.Ok; } else { // load bitmap and determine its quality // Parallel doesn't await async bodies so awaiting the load results in Parallel prematurely concluding the body task completed and // dispatching another, resulting in system overload. The simplest solution is to block this worker thread, which is OK as there // are many workers and they're decoupled from the UI thread. bitmapSource = file.LoadBitmapAsync(this.FolderPath, folderLoadProgress.RenderWidthBestEstimate).GetAwaiter().GetResult(); if (bitmapSource == Constant.Images.CorruptFile.Value) { file.ImageQuality = FileSelection.Corrupt; } else { file.ImageQuality = bitmapSource.AsWriteable().IsDark(this.state.DarkPixelThreshold, this.state.DarkPixelRatioThreshold); } } // see if the datetime can be updated from the metadata file.TryReadDateTimeOriginalFromMetadata(this.FolderPath, imageSetTimeZone); } catch (Exception exception) { Debug.Fail(String.Format("Load of {0} failed as it's likely corrupted.", file.FileName), exception.ToString()); bitmapSource = Constant.Images.CorruptFile.Value; file.ImageQuality = FileSelection.Corrupt; } lock (filesToInsert) { filesToInsert.Add(file); } DateTime utcNow = DateTime.UtcNow; if (utcNow - folderLoadProgress.MostRecentStatusDispatch > this.state.Throttles.DesiredIntervalBetweenRenders) { lock (folderLoadProgress) { if (utcNow - folderLoadProgress.MostRecentStatusDispatch > this.state.Throttles.DesiredIntervalBetweenRenders) { // if file was already loaded for dark checking use the resulting bitmap // otherwise, load the file for display if (bitmapSource != null) { folderLoadProgress.BitmapSource = bitmapSource; } else { folderLoadProgress.BitmapSource = null; } folderLoadProgress.CurrentFile = file; folderLoadProgress.CurrentFileIndex = filesToInsert.Count; folderLoadProgress.DisplayBitmap = true; folderLoadProgress.MostRecentStatusDispatch = utcNow; folderLoadStatus.Report(folderLoadProgress); } } } })); // Second pass: Update database // Parallel execution above produces out of order results. Put them back in order so the user sees images in file name order when // reviewing the image set. folderLoadProgress.DatabaseInsert = true; await Task.Run(() => { filesToInsert = filesToInsert.OrderBy(file => Path.Combine(file.RelativePath, file.FileName)).ToList(); this.dataHandler.FileDatabase.AddFiles(filesToInsert, (ImageRow file, int fileIndex) => { // skip reloading images to display as the user's already seen them import folderLoadProgress.BitmapSource = null; folderLoadProgress.CurrentFile = file; folderLoadProgress.CurrentFileIndex = fileIndex; folderLoadProgress.DisplayBitmap = false; folderLoadStatus.Report(folderLoadProgress); }); }); // hide the feedback bar, show the file slider this.FeedbackControl.Visibility = Visibility.Collapsed; this.FileNavigatorSlider.Visibility = Visibility.Visible; await this.OnFolderLoadingCompleteAsync(true); // tell the user how many files were loaded this.MaybeShowFileCountsDialog(true); return true; }