public static Task Run(Snapshots.ISnapshotService snapshot, Options options, BackupStatsCollector stats, BackupDatabase database)
{
return(AutomationExtensions.RunTask(
new
{
Input = Channels.ProcessedFiles.ForRead,
Output = Channels.AcceptedChangedFile.ForWrite
},
async self =>
{
var EMPTY_METADATA = Utility.WrapMetadata(new Dictionary <string, string>(), options);
var blocksize = options.Blocksize;
while (true)
{
var e = await self.Input.ReadAsync();
long filestatsize = -1;
try
{
filestatsize = snapshot.GetFileSize(e.Path);
}
catch (Exception ex)
{
Logging.Log.WriteExplicitMessage(FILELOGTAG, "FailedToReadSize", ex, "Failed tp read size of file: {0}", e.Path);
}
await stats.AddExaminedFile(filestatsize);
e.MetaHashAndSize = options.StoreMetadata ? Utility.WrapMetadata(await MetadataGenerator.GenerateMetadataAsync(e.Path, e.Attributes, options, snapshot), options) : EMPTY_METADATA;
var timestampChanged = e.LastWrite != e.OldModified || e.LastWrite.Ticks == 0 || e.OldModified.Ticks == 0;
var filesizeChanged = filestatsize < 0 || e.LastFileSize < 0 || filestatsize != e.LastFileSize;
var tooLargeFile = options.SkipFilesLargerThan != long.MaxValue && options.SkipFilesLargerThan != 0 && filestatsize >= 0 && filestatsize > options.SkipFilesLargerThan;
e.MetadataChanged = !options.CheckFiletimeOnly && !options.SkipMetadata && (e.MetaHashAndSize.Blob.Length != e.OldMetaSize || e.MetaHashAndSize.FileHash != e.OldMetaHash);
if ((e.OldId < 0 || options.DisableFiletimeCheck || timestampChanged || filesizeChanged || e.MetadataChanged) && !tooLargeFile)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "CheckFileForChanges", "Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", e.Path, e.OldId <= 0, timestampChanged, filesizeChanged, e.MetadataChanged, e.LastWrite, e.OldModified);
await self.Output.WriteAsync(e);
}
else
{
if (tooLargeFile)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckTooLarge", "Skipped checking file, because the size exceeds limit {0}", e.Path);
}
else
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Path);
}
await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
}
}
}));
}
private async Task RunAsync(string[] sources, Library.Utility.IFilter filter)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Begin);
// New isolated scope for each operation
using (new IsolatedChannelScope())
using (m_database = new LocalBackupDatabase(m_options.Dbpath, m_options))
{
m_result.SetDatabase(m_database);
m_result.Dryrun = m_options.Dryrun;
// Check the database integrity
Utility.UpdateOptionsFromDb(m_database, m_options);
Utility.VerifyParameters(m_database, m_options);
var probe_path = m_database.GetFirstPath();
if (probe_path != null && Duplicati.Library.Utility.Utility.GuessDirSeparator(probe_path) != System.IO.Path.DirectorySeparatorChar.ToString())
{
throw new UserInformationException(string.Format("The backup contains files that belong to another operating system. Proceeding with a backup would cause the database to contain paths from two different operation systems, which is not supported. To proceed without losing remote data, delete all filesets and make sure the --{0} option is set, then run the backup again to re-use the existing data on the remote store.", "no-auto-compact"), "CrossOsDatabaseReuseNotSupported");
}
if (m_database.PartiallyRecreated)
{
throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated");
}
if (m_database.RepairInProgress)
{
throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the backup process cannot continue. You may delete the local database and attempt to repair it again.", "DatabaseRepairInProgress");
}
// If there is no filter, we set an empty filter to simplify the code
// If there is a filter, we make sure that the sources are included
m_filter = filter ?? new Library.Utility.FilterExpression();
m_sourceFilter = new Library.Utility.FilterExpression(sources, true);
Task parallelScanner = null;
Task uploader = null;
try
{
// Setup runners and instances here
using (var db = new Backup.BackupDatabase(m_database, m_options))
using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, m_database))
using (var filesetvolume = new FilesetVolumeWriter(m_options, m_database.OperationTimestamp))
using (var stats = new Backup.BackupStatsCollector(m_result))
using (var bk = new Common.BackendHandler(m_options, m_backendurl, db, stats, m_result.TaskReader))
// Keep a reference to these channels to avoid shutdown
using (var uploadtarget = ChannelManager.GetChannel(Backup.Channels.BackendRequest.ForWrite))
{
long filesetid;
var counterToken = new CancellationTokenSource();
using (var snapshot = GetSnapshot(sources, m_options))
{
try
{
// Start parallel scan, or use the database
if (m_options.DisableFileScanner)
{
var d = m_database.GetLastBackupFileCountAndSize();
m_result.OperationProgressUpdater.UpdatefileCount(d.Item1, d.Item2, true);
}
else
{
parallelScanner = Backup.CountFilesHandler.Run(sources, snapshot, m_result, m_options, m_sourceFilter, m_filter, m_result.TaskReader, counterToken.Token);
}
// Make sure the database is sane
await db.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, true);
// Start the uploader process
uploader = Backup.BackendUploader.Run(bk, m_options, db, m_result, m_result.TaskReader, stats);
// If we have an interrupted backup, grab the
string lasttempfilelist = null;
long lasttempfileid = -1;
if (!m_options.DisableSyntheticFilelist)
{
var candidates = (await db.GetIncompleteFilesetsAsync()).OrderBy(x => x.Value).ToArray();
if (candidates.Length > 0)
{
lasttempfileid = candidates.Last().Key;
lasttempfilelist = m_database.GetRemoteVolumeFromID(lasttempfileid).Name;
}
}
// TODO: Rewrite to using the uploader process, or the BackendHandler interface
// Do a remote verification, unless disabled
PreBackupVerify(backend, lasttempfilelist);
// If the previous backup was interrupted, send a synthetic list
await Backup.UploadSyntheticFilelist.Run(db, m_options, m_result, m_result.TaskReader, lasttempfilelist, lasttempfileid);
// Grab the previous backup ID, if any
var prevfileset = m_database.FilesetTimes.FirstOrDefault();
if (prevfileset.Value.ToUniversalTime() > m_database.OperationTimestamp.ToUniversalTime())
{
throw new Exception(string.Format("The previous backup has time {0}, but this backup has time {1}. Something is wrong with the clock.", prevfileset.Value.ToLocalTime(), m_database.OperationTimestamp.ToLocalTime()));
}
var lastfilesetid = prevfileset.Value.Ticks == 0 ? -1 : prevfileset.Key;
// Rebuild any index files that are missing
await Backup.RecreateMissingIndexFiles.Run(db, m_options, m_result, m_result.TaskReader);
// This should be removed as the lookups are no longer used
m_database.BuildLookupTable(m_options);
// Prepare the operation by registering the filelist
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_ProcessingFiles);
var repcnt = 0;
while (repcnt < 100 && await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename) >= 0)
{
filesetvolume.ResetRemoteFilename(m_options, m_database.OperationTimestamp.AddSeconds(repcnt++));
}
if (await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename) >= 0)
{
throw new Exception("Unable to generate a unique fileset name");
}
var filesetvolumeid = await db.RegisterRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary);
filesetid = await db.CreateFilesetAsync(filesetvolumeid, VolumeBase.ParseFilename(filesetvolume.RemoteFilename).Time);
// create USN-based scanner if enabled
var journalService = GetJournalService(sources, snapshot, filter, lastfilesetid);
// Run the backup operation
if (await m_result.TaskReader.ProgressAsync)
{
await RunMainOperation(sources, snapshot, journalService, db, stats, m_options, m_sourceFilter, m_filter, m_result, m_result.TaskReader, lastfilesetid).ConfigureAwait(false);
}
}
finally
{
//If the scanner is still running for some reason, make sure we kill it now
counterToken.Cancel();
}
}
// Ensure the database is in a sane state after adding data
using (new Logging.Timer(LOGTAG, "VerifyConsistency", "VerifyConsistency"))
await db.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, false);
// Send the actual filelist
if (await m_result.TaskReader.ProgressAsync)
{
await Backup.UploadRealFilelist.Run(m_result, db, m_options, filesetvolume, filesetid, m_result.TaskReader);
}
// Wait for upload completion
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload);
var lastVolumeSize = await FlushBackend(m_result, uploadtarget, uploader).ConfigureAwait(false);
// Make sure we have the database up-to-date
await db.CommitTransactionAsync("CommitAfterUpload", false);
// TODO: Remove this later
m_transaction = m_database.BeginTransaction();
if (await m_result.TaskReader.ProgressAsync)
{
CompactIfRequired(backend, lastVolumeSize);
}
if (m_options.UploadVerificationFile && await m_result.TaskReader.ProgressAsync)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload);
FilelistProcessor.UploadVerificationFile(backend.BackendUrl, m_options, m_result.BackendWriter, m_database, m_transaction);
}
if (m_options.Dryrun)
{
m_transaction.Rollback();
m_transaction = null;
}
else
{
using (new Logging.Timer(LOGTAG, "CommitFinalizingBackup", "CommitFinalizingBackup"))
m_transaction.Commit();
m_transaction = null;
if (m_result.TaskControlRendevouz() != TaskControlState.Stop)
{
if (m_options.NoBackendverification)
{
UpdateStorageStatsFromDatabase();
}
else
{
PostBackupVerification();
}
}
}
m_database.WriteResults();
m_database.PurgeLogData(m_options.LogRetention);
if (m_options.AutoVacuum)
{
m_database.Vacuum();
}
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Complete);
return;
}
}
catch (Exception ex)
{
var aex = BuildException(ex, uploader, parallelScanner);
Logging.Log.WriteErrorMessage(LOGTAG, "FatalError", ex, "Fatal error");
if (aex == ex)
{
throw;
}
throw aex;
}
finally
{
if (parallelScanner != null && !parallelScanner.IsCompleted)
{
parallelScanner.Wait(500);
}
// TODO: We want to commit? always?
if (m_transaction != null)
{
try { m_transaction.Rollback(); }
catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "RollbackError", ex, "Rollback error: {0}", ex.Message); }
}
}
}
}
/// <summary>
/// Performs the bulk of work by starting all relevant processes
/// </summary>
private static async Task RunMainOperation(IEnumerable <string> sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, Backup.BackupDatabase database, Backup.BackupStatsCollector stats, Options options, IFilter sourcefilter, IFilter filter, BackupResults result, Common.ITaskReader taskreader, long lastfilesetid)
{
using (new Logging.Timer(LOGTAG, "BackupMainOperation", "BackupMainOperation"))
{
// Make sure the CompressionHints table is initialized, otherwise all workers will initialize it
var tb = options.CompressionHints.Count;
Task all;
using (new ChannelScope())
{
all = Task.WhenAll(
new []
{
Backup.DataBlockProcessor.Run(database, options, taskreader),
Backup.FileBlockProcessor.Run(snapshot, options, database, stats, taskreader),
Backup.StreamBlockSplitter.Run(options, database, taskreader),
Backup.FileEnumerationProcess.Run(sources, snapshot, journalService, options.FileAttributeFilter, sourcefilter, filter, options.SymlinkPolicy, options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames, options.ChangedFilelist, taskreader),
Backup.FilePreFilterProcess.Run(snapshot, options, stats, database),
Backup.MetadataPreProcess.Run(snapshot, options, database, lastfilesetid),
Backup.SpillCollectorProcess.Run(options, database, taskreader),
Backup.ProgressHandler.Run(result)
}
// Spawn additional block hashers
.Union(
Enumerable.Range(0, options.ConcurrencyBlockHashers - 1).Select(x => Backup.StreamBlockSplitter.Run(options, database, taskreader))
)
// Spawn additional compressors
.Union(
Enumerable.Range(0, options.ConcurrencyCompressors - 1).Select(x => Backup.DataBlockProcessor.Run(database, options, taskreader))
)
);
}
await all.ConfigureAwait(false);
if (options.ChangedFilelist != null && options.ChangedFilelist.Length >= 1)
{
await database.AppendFilesFromPreviousSetAsync(options.DeletedFilelist);
}
else if (journalService != null)
{
// append files from previous fileset, unless part of modifiedSources, which we've just scanned
await database.AppendFilesFromPreviousSetWithPredicateAsync((path, fileSize) =>
{
if (journalService.IsPathEnumerated(path))
{
return(true);
}
if (fileSize >= 0)
{
stats.AddExaminedFile(fileSize);
}
return(false);
});
// store journal data in database
var data = journalService.VolumeDataList.Where(p => p.JournalData != null).Select(p => p.JournalData).ToList();
if (data.Any())
{
// always record change journal data for current fileset (entry may be dropped later if nothing is uploaded)
await database.CreateChangeJournalDataAsync(data);
// update the previous fileset's change journal entry to resume at this point in case nothing was backed up
await database.UpdateChangeJournalDataAsync(data, lastfilesetid);
}
}
result.OperationProgressUpdater.UpdatefileCount(result.ExaminedFiles, result.SizeOfExaminedFiles, true);
}
}
public static Task Run(Snapshots.ISnapshotService snapshot, Options options, BackupStatsCollector stats, BackupDatabase database)
{
return(AutomationExtensions.RunTask(
new
{
Input = Channels.ProcessedFiles.ForRead,
ProgressChannel = Channels.ProgressEvents.ForWrite,
Output = Channels.AcceptedChangedFile.ForWrite
},
async self =>
{
var EMPTY_METADATA = Utility.WrapMetadata(new Dictionary <string, string>(), options);
// Pre-cache the option variables here to simplify and
// speed up repeated option access below
var SKIPFILESLARGERTHAN = options.SkipFilesLargerThan;
// Zero and max both indicate no size limit
if (SKIPFILESLARGERTHAN == long.MaxValue)
{
SKIPFILESLARGERTHAN = 0;
}
var DISABLEFILETIMECHECK = options.DisableFiletimeCheck;
var CHECKFILETIMEONLY = options.CheckFiletimeOnly;
var SKIPMETADATA = options.SkipMetadata;
while (true)
{
var e = await self.Input.ReadAsync();
long filestatsize = -1;
try
{
filestatsize = snapshot.GetFileSize(e.Path);
}
catch (Exception ex)
{
Logging.Log.WriteExplicitMessage(FILELOGTAG, "FailedToReadSize", ex, "Failed to read size of file: {0}", e.Path);
}
await stats.AddExaminedFile(filestatsize);
// Stop now if the file is too large
var tooLargeFile = SKIPFILESLARGERTHAN != 0 && filestatsize >= 0 && filestatsize > SKIPFILESLARGERTHAN;
if (tooLargeFile)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckTooLarge", "Skipped checking file, because the size exceeds limit {0}", e.Path);
await self.ProgressChannel.WriteAsync(new ProgressEvent()
{
Filepath = e.Path, Length = filestatsize, Type = EventType.FileSkipped
});
continue;
}
// Invalid ID indicates a new file
var isNewFile = e.OldId < 0;
// If we disable the filetime check, we always assume that the file has changed
// Otherwise we check that the timestamps are different or if any of them are empty
var timestampChanged = DISABLEFILETIMECHECK || e.LastWrite != e.OldModified || e.LastWrite.Ticks == 0 || e.OldModified.Ticks == 0;
// Avoid generating a new matadata blob if timestamp has not changed
// and we only check for timestamp changes
if (CHECKFILETIMEONLY && !timestampChanged && !isNewFile)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Path);
try
{
await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
}
catch (Exception ex)
{
if (ex.IsRetiredException())
{
throw;
}
Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Path);
}
await self.ProgressChannel.WriteAsync(new ProgressEvent()
{
Filepath = e.Path, Length = filestatsize, Type = EventType.FileSkipped
});
continue;
}
// If we have have disabled the filetime check, we do not have the metadata info
// but we want to know if the metadata is potentially changed
if (!isNewFile && DISABLEFILETIMECHECK)
{
var tp = await database.GetMetadataHashAndSizeForFileAsync(e.OldId);
if (tp != null)
{
e.OldMetaSize = tp.Item1;
e.OldMetaHash = tp.Item2;
}
}
// Compute current metadata
e.MetaHashAndSize = SKIPMETADATA ? EMPTY_METADATA : Utility.WrapMetadata(MetadataGenerator.GenerateMetadata(e.Path, e.Attributes, options, snapshot), options);
e.MetadataChanged = !SKIPMETADATA && (e.MetaHashAndSize.Blob.Length != e.OldMetaSize || e.MetaHashAndSize.FileHash != e.OldMetaHash);
// Check if the file is new, or something indicates a change
var filesizeChanged = filestatsize < 0 || e.LastFileSize < 0 || filestatsize != e.LastFileSize;
if (isNewFile || timestampChanged || filesizeChanged || e.MetadataChanged)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "CheckFileForChanges", "Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", e.Path, isNewFile, timestampChanged, filesizeChanged, e.MetadataChanged, e.LastWrite, e.OldModified);
await self.Output.WriteAsync(e);
}
else
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoMetadataChange", "Skipped checking file, because no metadata was updated {0}", e.Path);
try
{
await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Path);
}
await self.ProgressChannel.WriteAsync(new ProgressEvent()
{
Filepath = e.Path, Length = filestatsize, Type = EventType.FileSkipped
});
}
}
}));
}
public static Task Run(Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, BackupStatsCollector stats, ITaskReader taskreader)
{
return(AutomationExtensions.RunTask(
new
{
Input = Channels.AcceptedChangedFile.ForRead,
StreamBlockChannel = Channels.StreamBlock.ForWrite,
},
async self =>
{
var blocksize = options.Blocksize;
while (await taskreader.ProgressAsync)
{
var e = await self.Input.ReadAsync();
try
{
var hint = options.GetCompressionHintFromFilename(e.Path);
var oldHash = e.OldId < 0 ? null : await database.GetFileHashAsync(e.OldId);
StreamProcessResult filestreamdata;
// Process metadata and actual data in parallel
var metatask =
Task.Run(async() =>
{
// If we have determined that metadata has not changed, just grab the ID
if (!e.MetadataChanged)
{
var res = await database.GetMetadataIDAsync(e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length);
if (res.Item1)
{
return res.Item2;
}
Logging.Log.WriteWarningMessage(FILELOGTAG, "UnexpextedMetadataLookup", null, "Metadata was reported as not changed, but still requires being added?\nHash: {0}, Length: {1}, ID: {2}, Path: {3}", e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length, res.Item2, e.Path);
e.MetadataChanged = true;
}
return (await MetadataPreProcess.AddMetadataToOutputAsync(e.Path, e.MetaHashAndSize, database, self.StreamBlockChannel)).Item2;
});
using (var fs = snapshot.OpenRead(e.Path))
filestreamdata = await StreamBlock.ProcessStream(self.StreamBlockChannel, e.Path, fs, false, hint);
await stats.AddOpenedFile(filestreamdata.Streamlength);
var metadataid = await metatask;
var filekey = filestreamdata.Streamhash;
var filesize = filestreamdata.Streamlength;
if (oldHash != filekey)
{
if (oldHash == null)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "NewFile", "New file {0}", e.Path);
}
else
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "ChangedFile", "File has changed {0}", e.Path);
}
if (e.OldId < 0)
{
await stats.AddAddedFile(filesize);
if (options.Dryrun)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "WoudlAddNewFile", "Would add new file {0}, size {1}", e.Path, Library.Utility.Utility.FormatSizeString(filesize));
}
}
else
{
await stats.AddModifiedFile(filesize);
if (options.Dryrun)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "WoudlAddChangedFile", "Would add changed file {0}, size {1}", e.Path, Library.Utility.Utility.FormatSizeString(filesize));
}
}
await database.AddFileAsync(e.Path, e.LastWrite, filestreamdata.Blocksetid, metadataid);
}
else if (e.MetadataChanged)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileMetadataChanged", "File has only metadata changes {0}", e.Path);
await database.AddFileAsync(e.Path, e.LastWrite, filestreamdata.Blocksetid, metadataid);
}
else /*if (e.OldId >= 0)*/
{
// When we write the file to output, update the last modified time
Logging.Log.WriteVerboseMessage(FILELOGTAG, "NoFileChanges", "File has not changed {0}", e.Path);
try
{
await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Path);
}
}
}
catch (Exception ex)
{
if (ex.IsRetiredException())
{
return;
}
else
{
Logging.Log.WriteWarningMessage(FILELOGTAG, "PathProcessingFailed", ex, "Failed to process path: {0}", e.Path);
}
}
}
}
));
}
/// <summary>
/// Performs the bulk of work by starting all relevant processes
/// </summary>
private static async Task RunMainOperation(Snapshots.ISnapshotService snapshot, Backup.BackupDatabase database, Backup.BackupStatsCollector stats, Options options, IFilter sourcefilter, IFilter filter, BackupResults result, Common.ITaskReader taskreader, long lastfilesetid)
{
using (new Logging.Timer(LOGTAG, "BackupMainOperation", "BackupMainOperation"))
{
// Make sure the CompressionHints table is initialized, otherwise all workers will initialize it
var tb = options.CompressionHints.Count;
Task all;
using (new ChannelScope())
{
all = Task.WhenAll(
new []
{
Backup.DataBlockProcessor.Run(database, options, taskreader),
Backup.FileBlockProcessor.Run(snapshot, options, database, stats, taskreader),
Backup.StreamBlockSplitter.Run(options, database, taskreader),
Backup.FileEnumerationProcess.Run(snapshot, options.FileAttributeFilter, sourcefilter, filter, options.SymlinkPolicy, options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames, options.ChangedFilelist, taskreader),
Backup.FilePreFilterProcess.Run(snapshot, options, stats, database),
Backup.MetadataPreProcess.Run(snapshot, options, database, lastfilesetid),
Backup.SpillCollectorProcess.Run(options, database, taskreader),
Backup.ProgressHandler.Run(result)
}
// Spawn additional block hashers
.Union(
Enumerable.Range(0, options.ConcurrencyBlockHashers - 1).Select(x => Backup.StreamBlockSplitter.Run(options, database, taskreader))
)
// Spawn additional compressors
.Union(
Enumerable.Range(0, options.ConcurrencyCompressors - 1).Select(x => Backup.DataBlockProcessor.Run(database, options, taskreader))
)
);
}
await all;
if (options.ChangedFilelist != null && options.ChangedFilelist.Length >= 1)
{
await database.AppendFilesFromPreviousSetAsync(options.DeletedFilelist);
}
result.OperationProgressUpdater.UpdatefileCount(result.ExaminedFiles, result.SizeOfExaminedFiles, true);
}
}
