public static Task Run(BackupDatabase database, Options options, BackupResults result, ITaskReader taskreader)
{
return(AutomationExtensions.RunTask(new
{
UploadChannel = Channels.BackendRequest.ForWrite
},
async self =>
{
if (options.IndexfilePolicy != Options.IndexFileStrategy.None)
{
foreach (var blockfile in await database.GetMissingIndexFilesAsync())
{
if (!await taskreader.ProgressAsync)
{
return;
}
Logging.Log.WriteInformationMessage(LOGTAG, "RecreateMissingIndexFile", "Re-creating missing index file for {0}", blockfile);
var w = await Common.IndexVolumeCreator.CreateIndexVolume(blockfile, options, database);
if (!await taskreader.ProgressAsync)
{
return;
}
await database.UpdateRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
await self.UploadChannel.WriteAsync(new IndexVolumeUploadRequest(w));
}
}
}));
}
public BackupHandler(string backendurl, Options options, BackupResults results)
{
EMPTY_METADATA = Utility.WrapMetadata(new Dictionary<string, string>(), options);
m_options = options;
m_result = results;
m_backendurl = backendurl;
m_attributeFilter = m_options.FileAttributeFilter;
m_symlinkPolicy = m_options.SymlinkPolicy;
m_blocksize = m_options.Blocksize;
m_blockbuffer = new byte[m_options.Blocksize * Math.Max(1, m_options.FileReadBufferSize / m_options.Blocksize)];
m_blocklistbuffer = new byte[m_options.Blocksize];
m_blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
m_filehasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.FileHashAlgorithm);
if (m_blockhasher == null)
throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.BlockHashAlgorithm));
if (m_filehasher == null)
throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.FileHashAlgorithm));
if (!m_blockhasher.CanReuseTransform)
throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.BlockHashAlgorithm));
if (!m_filehasher.CanReuseTransform)
throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.FileHashAlgorithm));
if (options.AllowPassphraseChange)
throw new Exception(Strings.Foresthash.PassphraseChangeUnsupported);
}
internal void UpdateChangeStatistics(BackupResults results)
{
using (var cmd = m_connection.CreateCommand())
{
var lastFilesetId = GetPreviousFilesetID(cmd);
results.AddedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" = ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", m_filesetId, FOLDER_BLOCKSET_ID, lastFilesetId));
results.AddedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" = ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", m_filesetId, SYMLINK_BLOCKSET_ID, lastFilesetId));
results.DeletedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" = ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", lastFilesetId, FOLDER_BLOCKSET_ID, m_filesetId));
results.DeletedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" = ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", lastFilesetId, SYMLINK_BLOCKSET_ID, m_filesetId));
var subqueryNonFiles = @"SELECT ""File"".""Path"", ""Blockset"".""Fullhash"" FROM ""File"", ""FilesetEntry"", ""Metadataset"", ""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" AND ""File"".""BlocksetID"" = ? AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FilesetID"" = ? ";
results.ModifiedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM (" + subqueryNonFiles + @") A, (" + subqueryNonFiles + @") B WHERE ""A"".""Path"" = ""B"".""Path"" AND ""A"".""Fullhash"" != ""B"".""Fullhash"" ", lastFilesetId, FOLDER_BLOCKSET_ID, m_filesetId, FOLDER_BLOCKSET_ID));
results.ModifiedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM (" + subqueryNonFiles + @") A, (" + subqueryNonFiles + @") B WHERE ""A"".""Path"" = ""B"".""Path"" AND ""A"".""Fullhash"" != ""B"".""Fullhash"" ", lastFilesetId, SYMLINK_BLOCKSET_ID, m_filesetId, SYMLINK_BLOCKSET_ID));
var tmpName = "TmpFileList-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
try
{
var subqueryFiles = @"SELECT ""File"".""Path"", ""A"".""Fullhash"" AS ""Filehash"", ""B"".""Fullhash"" AS ""Metahash"" FROM ""File"", ""FilesetEntry"", ""Blockset"" A, ""Blockset"" B, ""Metadataset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""A"".""ID"" = ""File"".""BlocksetID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""B"".""ID"" ";
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS " + subqueryFiles, tmpName), lastFilesetId);
results.AddedFiles = Convert.ToInt64(cmd.ExecuteScalar(string.Format(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" != ? AND ""File"".""BlocksetID"" != ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""{0}"")", tmpName), m_filesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID));
results.DeletedFiles = Convert.ToInt64(cmd.ExecuteScalar(string.Format(@"SELECT COUNT(*) FROM ""{0}"" WHERE ""{0}"".""Path"" NOT IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", tmpName), m_filesetId));
results.ModifiedFiles = Convert.ToInt64(cmd.ExecuteScalar(string.Format(@"SELECT COUNT(*) FROM ""{0}"" A, (" + subqueryFiles + @") B WHERE ""A"".""Path"" = ""B"".""Path"" AND (""A"".""Filehash"" != ""B"".""Filehash"" OR ""A"".""Metahash"" != ""B"".""Metahash"")", tmpName), m_filesetId));
}
finally
{
try { cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"";", tmpName)); }
catch (Exception ex) { m_result.AddWarning("Dispose temp table error", ex); }
}
}
}
public BackupHandler(string backendurl, Options options, BackupResults results)
{
m_options = options;
m_result = results;
m_backendurl = backendurl;
if (options.AllowPassphraseChange)
{
throw new UserInformationException(Strings.Common.PassphraseChangeUnsupported, "PassphraseChangeUnsupported");
}
}
private static async Task <long> FlushBackend(BackupResults result, IWriteChannel <Backup.IUploadRequest> uploadtarget, Task uploader)
{
var flushReq = new Backup.FlushRequest();
// Wait for upload completion
result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload);
await uploadtarget.WriteAsync(flushReq).ConfigureAwait(false);
await uploader.ConfigureAwait(false);
// Grab the size of the last uploaded volume
return(await flushReq.LastWriteSizeAsync);
}
public BackupHandler(string backendurl, Options options, BackupResults results)
{
EMPTY_METADATA = Utility.WrapMetadata(new Dictionary<string, string>(), options);
m_options = options;
m_result = results;
m_backendurl = backendurl;
m_attributeFilter = m_options.FileAttributeFilter;
m_symlinkPolicy = m_options.SymlinkPolicy;
if (options.AllowPassphraseChange)
throw new Exception(Strings.Foresthash.PassphraseChangeUnsupported);
}
public static Task Run(Snapshots.ISnapshotService snapshot, BackupResults result, Options options, IFilter sourcefilter, IFilter filter, Common.ITaskReader taskreader, System.Threading.CancellationToken token)
{
// Make sure we create the enumeration process in a seperate scope,
// but keep the log channel from the parent scope
using (Logging.Log.StartIsolatingScope())
using (new IsolatedChannelScope())
{
var enumeratorTask = Backup.FileEnumerationProcess.Run(snapshot, options.FileAttributeFilter, sourcefilter, filter, options.SymlinkPolicy, options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames, options.ChangedFilelist, taskreader);
var counterTask = AutomationExtensions.RunTask(new
{
Input = Backup.Channels.SourcePaths.ForRead
},
async self =>
{
var count = 0L;
var size = 0L;
try
{
while (await taskreader.ProgressAsync && !token.IsCancellationRequested)
{
var path = await self.Input.ReadAsync();
count++;
try
{
size += snapshot.GetFileSize(path);
}
catch
{
}
result.OperationProgressUpdater.UpdatefileCount(count, size, false);
}
}
finally
{
result.OperationProgressUpdater.UpdatefileCount(count, size, true);
}
});
return(Task.WhenAll(enumeratorTask, counterTask));
}
}
/// <summary>
/// Performs the bulk of work by starting all relevant processes
/// </summary>
private static async Task RunMainOperation(IEnumerable <string> sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, Backup.BackupDatabase database, Backup.BackupStatsCollector stats, Options options, IFilter sourcefilter, IFilter filter, BackupResults result, Common.ITaskReader taskreader, long lastfilesetid)
{
using (new Logging.Timer(LOGTAG, "BackupMainOperation", "BackupMainOperation"))
{
// Make sure the CompressionHints table is initialized, otherwise all workers will initialize it
var tb = options.CompressionHints.Count;
Task all;
using (new ChannelScope())
{
all = Task.WhenAll(
new []
{
Backup.DataBlockProcessor.Run(database, options, taskreader),
Backup.FileBlockProcessor.Run(snapshot, options, database, stats, taskreader),
Backup.StreamBlockSplitter.Run(options, database, taskreader),
Backup.FileEnumerationProcess.Run(sources, snapshot, journalService, options.FileAttributeFilter, sourcefilter, filter, options.SymlinkPolicy, options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames, options.ChangedFilelist, taskreader),
Backup.FilePreFilterProcess.Run(snapshot, options, stats, database),
Backup.MetadataPreProcess.Run(snapshot, options, database, lastfilesetid),
Backup.SpillCollectorProcess.Run(options, database, taskreader),
Backup.ProgressHandler.Run(result)
}
// Spawn additional block hashers
.Union(
Enumerable.Range(0, options.ConcurrencyBlockHashers - 1).Select(x => Backup.StreamBlockSplitter.Run(options, database, taskreader))
)
// Spawn additional compressors
.Union(
Enumerable.Range(0, options.ConcurrencyCompressors - 1).Select(x => Backup.DataBlockProcessor.Run(database, options, taskreader))
)
);
}
await all.ConfigureAwait(false);
if (options.ChangedFilelist != null && options.ChangedFilelist.Length >= 1)
{
await database.AppendFilesFromPreviousSetAsync(options.DeletedFilelist);
}
else if (journalService != null)
{
// append files from previous fileset, unless part of modifiedSources, which we've just scanned
await database.AppendFilesFromPreviousSetWithPredicateAsync((path, fileSize) =>
{
if (journalService.IsPathEnumerated(path))
{
return(true);
}
if (fileSize >= 0)
{
stats.AddExaminedFile(fileSize);
}
return(false);
});
// store journal data in database
var data = journalService.VolumeDataList.Where(p => p.JournalData != null).Select(p => p.JournalData).ToList();
if (data.Any())
{
// always record change journal data for current fileset (entry may be dropped later if nothing is uploaded)
await database.CreateChangeJournalDataAsync(data);
// update the previous fileset's change journal entry to resume at this point in case nothing was backed up
await database.UpdateChangeJournalDataAsync(data, lastfilesetid);
}
}
result.OperationProgressUpdater.UpdatefileCount(result.ExaminedFiles, result.SizeOfExaminedFiles, true);
}
}
public Task UpdateChangeStatisticsAsync(BackupResults result)
{
return(RunOnMain(() => m_database.UpdateChangeStatistics(result, m_transaction)));
}
public static Task Run(Common.BackendHandler backend, Options options, Common.DatabaseCommon database, BackupResults results, Common.ITaskReader taskreader, StatsCollector stats)
{
return(AutomationExtensions.RunTask(new
{
Input = Channels.BackendRequest.ForRead,
},
async self =>
{
var inProgress = new Queue <KeyValuePair <int, Task> >();
var max_pending = options.AsynchronousUploadLimit == 0 ? long.MaxValue : options.AsynchronousUploadLimit;
var noIndexFiles = options.IndexfilePolicy == Options.IndexFileStrategy.None;
var active = 0;
var lastSize = -1L;
while (!await self.Input.IsRetiredAsync && await taskreader.ProgressAsync)
{
try
{
var req = await self.Input.ReadAsync();
if (!await taskreader.ProgressAsync)
{
continue;
}
var task = default(KeyValuePair <int, Task>);
if (req is VolumeUploadRequest)
{
lastSize = ((VolumeUploadRequest)req).BlockVolume.SourceSize;
if (noIndexFiles || ((VolumeUploadRequest)req).IndexVolume == null)
{
task = new KeyValuePair <int, Task>(1, backend.UploadFileAsync(((VolumeUploadRequest)req).BlockVolume, null));
}
else
{
task = new KeyValuePair <int, Task>(2, backend.UploadFileAsync(((VolumeUploadRequest)req).BlockVolume, name => ((VolumeUploadRequest)req).IndexVolume.CreateVolume(name, options, database)));
}
}
else if (req is FilesetUploadRequest)
{
task = new KeyValuePair <int, Task>(1, backend.UploadFileAsync(((FilesetUploadRequest)req).Fileset));
}
else if (req is IndexVolumeUploadRequest)
{
task = new KeyValuePair <int, Task>(1, backend.UploadFileAsync(((IndexVolumeUploadRequest)req).IndexVolume));
}
else if (req is FlushRequest)
{
try
{
while (inProgress.Count > 0)
{
await inProgress.Dequeue().Value;
}
active = 0;
}
finally
{
((FlushRequest)req).SetFlushed(lastSize);
}
}
if (task.Value != null)
{
inProgress.Enqueue(task);
active += task.Key;
}
}
catch (Exception ex)
{
if (!ex.IsRetiredException())
{
throw;
}
}
while (active >= max_pending)
{
var top = inProgress.Dequeue();
// See if we are done
if (await Task.WhenAny(top.Value, Task.Delay(500)) != top.Value)
{
try
{
stats.SetBlocking(true);
await top.Value;
}
finally
{
stats.SetBlocking(false);
}
}
active -= top.Key;
}
}
results.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload);
try
{
stats.SetBlocking(true);
while (inProgress.Count > 0)
{
await inProgress.Dequeue().Value;
}
}
finally
{
stats.SetBlocking(false);
}
}));
}
public static Task Run(BackupDatabase database, Options options, BackupResults result, ITaskReader taskreader, string lasttempfilelist, long lasttempfileid)
{
return(AutomationExtensions.RunTask(new
{
UploadChannel = Channels.BackendRequest.ForWrite
},
async self =>
{
// Check if we should upload a synthetic filelist
if (options.DisableSyntheticFilelist || string.IsNullOrWhiteSpace(lasttempfilelist) || lasttempfileid < 0)
{
return;
}
// Check that we still need to process this after the cleanup has performed its duties
var syntbase = await database.GetRemoteVolumeFromIDAsync(lasttempfileid);
// If we do not have a valid entry, warn and quit
if (syntbase.Name == null || syntbase.State != RemoteVolumeState.Uploaded)
{
// TODO: If the repair succeeds, this could give a false warning?
Logging.Log.WriteWarningMessage(LOGTAG, "MissingTemporaryFilelist", null, "Expected there to be a temporary fileset for synthetic filelist ({0}, {1}), but none was found?", lasttempfileid, lasttempfilelist);
return;
}
// Files is missing or repaired
if (syntbase.Name == null || (syntbase.State != RemoteVolumeState.Uploading && syntbase.State != RemoteVolumeState.Temporary))
{
Logging.Log.WriteInformationMessage(LOGTAG, "SkippingSyntheticListUpload", "Skipping synthetic upload because temporary fileset appers to be complete: ({0}, {1}, {2})", lasttempfileid, lasttempfilelist, syntbase.State);
return;
}
// Ready to build and upload the synthetic list
await database.CommitTransactionAsync("PreSyntheticFilelist");
var incompleteFilesets = (await database.GetIncompleteFilesetsAsync()).OrderBy(x => x.Value).ToList();
result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PreviousBackupFinalize);
Logging.Log.WriteInformationMessage(LOGTAG, "PreviousBackupFilelistUpload", "Uploading filelist from previous interrupted backup");
if (!await taskreader.ProgressAsync)
{
return;
}
var incompleteSet = incompleteFilesets.Last();
var badIds = from n in incompleteFilesets select n.Key;
var prevs = (from n in await database.GetFilesetTimesAsync()
where
n.Key < incompleteSet.Key
&&
!badIds.Contains(n.Key)
orderby n.Key
select n.Key).ToArray();
var prevId = prevs.Length == 0 ? -1 : prevs.Last();
FilesetVolumeWriter fsw = null;
try
{
var s = 1;
var fileTime = incompleteSet.Value + TimeSpan.FromSeconds(s);
var oldFilesetID = incompleteSet.Key;
// Probe for an unused filename
while (s < 60)
{
var id = await database.GetRemoteVolumeIDAsync(VolumeBase.GenerateFilename(RemoteVolumeType.Files, options, null, fileTime));
if (id < 0)
{
break;
}
fileTime = incompleteSet.Value + TimeSpan.FromSeconds(++s);
}
fsw = new FilesetVolumeWriter(options, fileTime);
fsw.VolumeID = await database.RegisterRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary);
if (!string.IsNullOrEmpty(options.ControlFiles))
{
foreach (var p in options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
{
fsw.AddControlFile(p, options.GetCompressionHintFromFilename(p));
}
}
var newFilesetID = await database.CreateFilesetAsync(fsw.VolumeID, fileTime);
await database.LinkFilesetToVolumeAsync(newFilesetID, fsw.VolumeID);
await database.AppendFilesFromPreviousSetAsync(null, newFilesetID, prevId, fileTime);
await database.WriteFilesetAsync(fsw, newFilesetID);
if (!await taskreader.ProgressAsync)
{
return;
}
await database.UpdateRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
await database.CommitTransactionAsync("CommitUpdateFilelistVolume");
await self.UploadChannel.WriteAsync(new FilesetUploadRequest(fsw));
fsw = null;
}
catch
{
await database.RollbackTransactionAsync();
throw;
}
finally
{
if (fsw != null)
{
try { fsw.Dispose(); }
catch { fsw = null; }
}
}
}
));
}
public static Task Run(BackupResults result, BackupDatabase db, Options options, FilesetVolumeWriter filesetvolume, long filesetid, Common.ITaskReader taskreader)
{
return(AutomationExtensions.RunTask(new
{
Output = Channels.BackendRequest.ForWrite,
},
async self =>
{
if (!await taskreader.ProgressAsync)
{
return;
}
// Update the reported source and backend changes
using (new Logging.Timer(LOGTAG, "UpdateChangeStatistics", "UpdateChangeStatistics"))
await db.UpdateChangeStatisticsAsync(result);
var changeCount =
result.AddedFiles + result.ModifiedFiles + result.DeletedFiles +
result.AddedFolders + result.ModifiedFolders + result.DeletedFolders +
result.AddedSymlinks + result.ModifiedSymlinks + result.DeletedSymlinks;
//Changes in the filelist triggers a filelist upload
if (options.UploadUnchangedBackups || changeCount > 0)
{
using (new Logging.Timer(LOGTAG, "UploadNewFileset", "Uploading a new fileset"))
{
if (!string.IsNullOrEmpty(options.ControlFiles))
{
foreach (var p in options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
{
filesetvolume.AddControlFile(p, options.GetCompressionHintFromFilename(p));
}
}
if (!await taskreader.ProgressAsync)
{
return;
}
await db.WriteFilesetAsync(filesetvolume, filesetid);
filesetvolume.Close();
if (!await taskreader.ProgressAsync)
{
return;
}
await db.UpdateRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
await db.CommitTransactionAsync("CommitUpdateRemoteVolume");
await self.Output.WriteAsync(new FilesetUploadRequest(filesetvolume));
}
}
else
{
Logging.Log.WriteVerboseMessage(LOGTAG, "RemovingLeftoverTempFile", "removing temp files, as no data needs to be uploaded");
await db.RemoveRemoteVolumeAsync(filesetvolume.RemoteFilename);
}
await db.CommitTransactionAsync("CommitUpdateRemoteVolume");
}));
}
/// <summary>
/// Performs the bulk of work by starting all relevant processes
/// </summary>
private static async Task RunMainOperation(Snapshots.ISnapshotService snapshot, Backup.BackupDatabase database, Backup.BackupStatsCollector stats, Options options, IFilter sourcefilter, IFilter filter, BackupResults result, Common.ITaskReader taskreader, long lastfilesetid)
{
using (new Logging.Timer(LOGTAG, "BackupMainOperation", "BackupMainOperation"))
{
// Make sure the CompressionHints table is initialized, otherwise all workers will initialize it
var tb = options.CompressionHints.Count;
Task all;
using (new ChannelScope())
{
all = Task.WhenAll(
new []
{
Backup.DataBlockProcessor.Run(database, options, taskreader),
Backup.FileBlockProcessor.Run(snapshot, options, database, stats, taskreader),
Backup.StreamBlockSplitter.Run(options, database, taskreader),
Backup.FileEnumerationProcess.Run(snapshot, options.FileAttributeFilter, sourcefilter, filter, options.SymlinkPolicy, options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames, options.ChangedFilelist, taskreader),
Backup.FilePreFilterProcess.Run(snapshot, options, stats, database),
Backup.MetadataPreProcess.Run(snapshot, options, database, lastfilesetid),
Backup.SpillCollectorProcess.Run(options, database, taskreader),
Backup.ProgressHandler.Run(result)
}
// Spawn additional block hashers
.Union(
Enumerable.Range(0, options.ConcurrencyBlockHashers - 1).Select(x => Backup.StreamBlockSplitter.Run(options, database, taskreader))
)
// Spawn additional compressors
.Union(
Enumerable.Range(0, options.ConcurrencyCompressors - 1).Select(x => Backup.DataBlockProcessor.Run(database, options, taskreader))
)
);
}
await all;
if (options.ChangedFilelist != null && options.ChangedFilelist.Length >= 1)
{
await database.AppendFilesFromPreviousSetAsync(options.DeletedFilelist);
}
result.OperationProgressUpdater.UpdatefileCount(result.ExaminedFiles, result.SizeOfExaminedFiles, true);
}
}
public static Task Run(BackupResults stat)
{
return(AutomationExtensions.RunTask(new
{
Input = Channels.ProgressEvents.ForRead
},
async self =>
{
var filesStarted = new Dictionary <string, long>();
var fileProgress = new Dictionary <string, long>();
long processedFileCount = 0;
long processedFileSize = 0;
string current = null;
while (true)
{
var t = await self.Input.ReadAsync();
switch (t.Type)
{
case EventType.FileStarted:
filesStarted[t.Filepath] = t.Length;
fileProgress[t.Filepath] = 0;
break;
case EventType.FileProgressUpdate:
if (t.Filepath == current)
{
stat.OperationProgressUpdater.UpdateFileProgress(t.Length);
}
break;
case EventType.FileClosed:
if (fileProgress.ContainsKey(t.Filepath))
{
fileProgress[t.Filepath] = t.Length;
}
if (t.Filepath == current)
{
stat.OperationProgressUpdater.UpdateFileProgress(t.Length);
current = null;
}
processedFileCount += 1;
processedFileSize += t.Length;
stat.OperationProgressUpdater.UpdatefilesProcessed(processedFileCount, processedFileSize);
filesStarted.Remove(t.Filepath);
fileProgress.Remove(t.Filepath);
break;
case EventType.FileSkipped:
processedFileCount += 1;
processedFileSize += t.Length;
stat.OperationProgressUpdater.UpdatefilesProcessed(processedFileCount, processedFileSize);
break;
}
if (current == null)
{
current = filesStarted.OrderByDescending(x => x.Value).Select(x => x.Key).FirstOrDefault();
if (current != null)
{
stat.OperationProgressUpdater.StartFile(current, filesStarted[current]);
if (fileProgress.ContainsKey(current) && fileProgress[current] > 0)
{
stat.OperationProgressUpdater.UpdateFileProgress(fileProgress[current]);
}
}
}
}
}));
}
public BackupStatsCollector(BackupResults res)
: base(res.BackendWriter)
{
m_res = res;
}
