public static async Task <IndexVolumeWriter> CreateIndexVolume(string blockname, Options options, Common.DatabaseCommon database)
{
using (var h = Duplicati.Library.Utility.HashAlgorithmHelper.Create(options.BlockHashAlgorithm))
{
var w = new IndexVolumeWriter(options);
w.VolumeID = await database.RegisterRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary);
var blockvolume = await database.GetVolumeInfoAsync(blockname);
w.StartVolume(blockname);
foreach (var b in await database.GetBlocksAsync(blockvolume.ID))
{
w.AddBlock(b.Hash, b.Size);
}
w.FinishVolume(blockvolume.Hash, blockvolume.Size);
if (options.IndexfilePolicy == Options.IndexFileStrategy.Full)
{
foreach (var b in await database.GetBlocklistsAsync(blockvolume.ID, options.Blocksize, options.BlockhashSize))
{
var bh = Convert.ToBase64String(h.ComputeHash(b.Item2, 0, b.Item3));
if (bh != b.Item1)
{
throw new Exception(string.Format("Internal consistency check failed, generated index block has wrong hash, {0} vs {1}", bh, b.Item1));
}
w.WriteBlocklist(b.Item1, b.Item2, 0, b.Item3);
}
}
w.Close();
await database.AddIndexBlockLinkAsync(w.VolumeID, blockvolume.ID);
return(w);
}
}
public static Task Run(Common.BackendHandler backend, Options options, Common.DatabaseCommon database, BackupResults results, Common.ITaskReader taskreader, StatsCollector stats)
{
return(AutomationExtensions.RunTask(new
{
Input = Channels.BackendRequest.ForRead,
},
async self =>
{
var inProgress = new Queue <KeyValuePair <int, Task> >();
var max_pending = options.AsynchronousUploadLimit == 0 ? long.MaxValue : options.AsynchronousUploadLimit;
var noIndexFiles = options.IndexfilePolicy == Options.IndexFileStrategy.None;
var active = 0;
var lastSize = -1L;
while (!await self.Input.IsRetiredAsync && await taskreader.ProgressAsync)
{
try
{
var req = await self.Input.ReadAsync();
if (!await taskreader.ProgressAsync)
{
continue;
}
var task = default(KeyValuePair <int, Task>);
if (req is VolumeUploadRequest)
{
lastSize = ((VolumeUploadRequest)req).BlockVolume.SourceSize;
if (noIndexFiles || ((VolumeUploadRequest)req).IndexVolume == null)
{
task = new KeyValuePair <int, Task>(1, backend.UploadFileAsync(((VolumeUploadRequest)req).BlockVolume, null));
}
else
{
task = new KeyValuePair <int, Task>(2, backend.UploadFileAsync(((VolumeUploadRequest)req).BlockVolume, name => ((VolumeUploadRequest)req).IndexVolume.CreateVolume(name, options, database)));
}
}
else if (req is FilesetUploadRequest)
{
task = new KeyValuePair <int, Task>(1, backend.UploadFileAsync(((FilesetUploadRequest)req).Fileset));
}
else if (req is IndexVolumeUploadRequest)
{
task = new KeyValuePair <int, Task>(1, backend.UploadFileAsync(((IndexVolumeUploadRequest)req).IndexVolume));
}
else if (req is FlushRequest)
{
try
{
while (inProgress.Count > 0)
{
await inProgress.Dequeue().Value;
}
active = 0;
}
finally
{
((FlushRequest)req).SetFlushed(lastSize);
}
}
if (task.Value != null)
{
inProgress.Enqueue(task);
active += task.Key;
}
}
catch (Exception ex)
{
if (!ex.IsRetiredException())
{
throw;
}
}
while (active >= max_pending)
{
var top = inProgress.Dequeue();
// See if we are done
if (await Task.WhenAny(top.Value, Task.Delay(500)) != top.Value)
{
try
{
stats.SetBlocking(true);
await top.Value;
}
finally
{
stats.SetBlocking(false);
}
}
active -= top.Key;
}
}
results.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload);
try
{
stats.SetBlocking(true);
while (inProgress.Count > 0)
{
await inProgress.Dequeue().Value;
}
}
finally
{
stats.SetBlocking(false);
}
}));
}
/// <summary>
/// Creates an index volume with the temporary contents
/// </summary>
/// <returns>The index volume.</returns>
/// <param name="blockfilename">The name of the block file.</param>
public async Task <IndexVolumeWriter> CreateVolume(string blockfilename, Options options, Common.DatabaseCommon database)
{
var w = new IndexVolumeWriter(options);
w.VolumeID = await database.RegisterRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary);
var blockvolume = await database.GetVolumeInfoAsync(blockfilename);
w.StartVolume(blockfilename);
foreach (var n in blockHashes)
{
var args = n.Split(new char[] { ':' }, 2);
w.AddBlock(args[1], long.Parse(args[0]));
}
w.FinishVolume(blockvolume.Hash, blockvolume.Size);
var enumerator = blockListHashes.GetEnumerator();
while (enumerator.MoveNext())
{
var hash = enumerator.Current;
enumerator.MoveNext();
var data = Convert.FromBase64String(enumerator.Current);
w.WriteBlocklist(hash, data, 0, data.Length);
}
w.Close();
return(w);
}
