private async Task UploadBlockAndIndexAsync(VolumeUploadRequest upload, Worker worker, CancellationToken cancelToken) { var blockUploaded = await UploadFileAsync(upload.BlockEntry, worker, cancelToken).ConfigureAwait(false); if (blockUploaded && await UploadFileAsync(upload.IndexEntry, worker, cancelToken).ConfigureAwait(false)) { await m_database.AddIndexBlockLinkAsync(upload.IndexVolume.VolumeID, upload.BlockVolume.VolumeID).ConfigureAwait(false); } }
private async Task UploadBlockAndIndexAsync(VolumeUploadRequest upload, Worker worker, CancellationToken cancelToken) { if (await UploadFileAsync(upload.BlockEntry, worker, cancelToken).ConfigureAwait(false)) { // We must use the BlockEntry's RemoteFilename and not the BlockVolume's since the // BlockEntry's' RemoteFilename reflects renamed files due to retries after errors. IndexVolumeWriter indexVolumeWriter = await upload.IndexVolume.CreateVolume(upload.BlockEntry.RemoteFilename, upload.BlockEntry.Hash, upload.BlockEntry.Size, upload.Options, upload.Database); FileEntryItem indexEntry = indexVolumeWriter.CreateFileEntryForUpload(upload.Options); if (await UploadFileAsync(indexEntry, worker, cancelToken).ConfigureAwait(false)) { await m_database.AddIndexBlockLinkAsync(indexVolumeWriter.VolumeID, upload.BlockVolume.VolumeID).ConfigureAwait(false); } } }
private static async Task UploadVolumeAndIndex(SpillVolumeRequest target, IWriteChannel <IUploadRequest> outputChannel, Options options, BackupDatabase database) { var blockEntry = target.BlockVolume.CreateFileEntryForUpload(options); TemporaryIndexVolume indexVolumeCopy = null; if (target.IndexVolume != null) { indexVolumeCopy = new TemporaryIndexVolume(options); target.IndexVolume.CopyTo(indexVolumeCopy, false); } var uploadRequest = new VolumeUploadRequest(target.BlockVolume, blockEntry, indexVolumeCopy, options, database); await outputChannel.WriteAsync(uploadRequest).ConfigureAwait(false); }
private static async Task UploadVolumeAndIndex(SpillVolumeRequest target, IWriteChannel <IUploadRequest> outputChannel, Options options, BackupDatabase database) { var blockEntry = CreateFileEntryForUpload(target.BlockVolume, options); IndexVolumeWriter indexVolume = null; FileEntryItem indexEntry = null; if (target.IndexVolume != null) { indexVolume = await target.IndexVolume.CreateVolume(target.BlockVolume.RemoteFilename, blockEntry.Hash, blockEntry.Size, options, database).ConfigureAwait(false); indexEntry = CreateFileEntryForUpload(indexVolume, options); } var uploadRequest = new VolumeUploadRequest(target.BlockVolume, blockEntry, indexVolume, indexEntry); await outputChannel.WriteAsync(uploadRequest).ConfigureAwait(false); }
public static Task Run(BackupDatabase database, Options options, ITaskReader taskreader) { return(AutomationExtensions.RunTask( new { Input = Channels.OutputBlocks.ForRead, Output = Channels.BackendRequest.ForWrite, SpillPickup = Channels.SpillPickup.ForWrite, }, async self => { var noIndexFiles = options.IndexfilePolicy == Options.IndexFileStrategy.None; var fullIndexFiles = options.IndexfilePolicy == Options.IndexFileStrategy.Full; BlockVolumeWriter blockvolume = null; TemporaryIndexVolume indexvolume = null; try { while (true) { var b = await self.Input.ReadAsync(); // Lazy-start a new block volume if (blockvolume == null) { // Before we start a new volume, probe to see if it exists // This will delay creation of volumes for differential backups // There can be a race, such that two workers determine that // the block is missing, but this will be solved by the AddBlock call // which runs atomically if (await database.FindBlockIDAsync(b.HashKey, b.Size) >= 0) { b.TaskCompletion.TrySetResult(false); continue; } blockvolume = new BlockVolumeWriter(options); blockvolume.VolumeID = await database.RegisterRemoteVolumeAsync(blockvolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary); indexvolume = noIndexFiles ? null : new TemporaryIndexVolume(options); } var newBlock = await database.AddBlockAsync(b.HashKey, b.Size, blockvolume.VolumeID); b.TaskCompletion.TrySetResult(newBlock); if (newBlock) { blockvolume.AddBlock(b.HashKey, b.Data, b.Offset, (int)b.Size, b.Hint); if (indexvolume != null) { indexvolume.AddBlock(b.HashKey, b.Size); if (b.IsBlocklistHashes && fullIndexFiles) { indexvolume.AddBlockListHash(b.HashKey, b.Size, b.Data); } } // If the volume is full, send to upload if (blockvolume.Filesize > options.VolumeSize - options.Blocksize) { //When uploading a new volume, we register the volumes and then flush the transaction // this ensures that the local database and remote storage are as closely related as possible await database.UpdateRemoteVolumeAsync(blockvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null); blockvolume.Close(); await database.CommitTransactionAsync("CommitAddBlockToOutputFlush"); FileEntryItem blockEntry = blockvolume.CreateFileEntryForUpload(options); TemporaryIndexVolume indexVolumeCopy = null; if (indexvolume != null) { indexVolumeCopy = new TemporaryIndexVolume(options); indexvolume.CopyTo(indexVolumeCopy, false); } var uploadRequest = new VolumeUploadRequest(blockvolume, blockEntry, indexVolumeCopy, options, database); blockvolume = null; indexvolume = null; // Write to output at the end here to prevent sending a full volume to the SpillCollector await self.Output.WriteAsync(uploadRequest); } } // We ignore the stop signal, but not the pause and terminate await taskreader.ProgressAsync; } } catch (Exception ex) { if (ex.IsRetiredException()) { // If we have collected data, merge all pending volumes into a single volume if (blockvolume != null && blockvolume.SourceSize > 0) { await self.SpillPickup.WriteAsync(new SpillVolumeRequest(blockvolume, indexvolume)); } } throw; } })); }
public static Task Run(Options options, BackupDatabase database, ITaskReader taskreader) { return(AutomationExtensions.RunTask( new { Input = Channels.SpillPickup.ForRead, Output = Channels.BackendRequest.ForWrite, }, async self => { var lst = new List <VolumeUploadRequest>(); while (!await self.Input.IsRetiredAsync) { try { lst.Add((VolumeUploadRequest)await self.Input.ReadAsync()); } catch (Exception ex) { if (ex.IsRetiredException()) { break; } throw; } } while (lst.Count > 1) { // We ignore the stop signal, but not the pause and terminate await taskreader.ProgressAsync; VolumeUploadRequest target = null; var source = lst[0]; // Finalize the current work source.BlockVolume.Close(); // Remove it from the list of active operations lst.RemoveAt(0); var buffer = new byte[options.Blocksize]; using (var rd = new BlockVolumeReader(options.CompressionModule, source.BlockVolume.LocalFilename, options)) { foreach (var file in rd.Blocks) { // Grab a target if (target == null) { if (lst.Count == 0) { // No more targets, make one target = new VolumeUploadRequest(new BlockVolumeWriter(options), source.IndexVolume == null ? null : new TemporaryIndexVolume(options)); target.BlockVolume.VolumeID = await database.RegisterRemoteVolumeAsync(target.BlockVolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary); } else { // Grab the next target target = lst[0]; lst.RemoveAt(0); } // We copy all the blocklisthashes, which may create duplicates // but otherwise we need to query all hashes to see if they are blocklisthashes if (source.IndexVolume != null) { source.IndexVolume.CopyTo(target.IndexVolume, true); } } var len = rd.ReadBlock(file.Key, buffer); target.BlockVolume.AddBlock(file.Key, buffer, 0, len, Duplicati.Library.Interface.CompressionHint.Default); await database.MoveBlockToVolumeAsync(file.Key, len, source.BlockVolume.VolumeID, target.BlockVolume.VolumeID); if (target.IndexVolume != null) { target.IndexVolume.AddBlock(file.Key, len); } if (target.BlockVolume.Filesize > options.VolumeSize - options.Blocksize) { target.BlockVolume.Close(); await self.Output.WriteAsync(target); target = null; } } } // Make sure they are out of the database System.IO.File.Delete(source.BlockVolume.LocalFilename); await database.SafeDeleteRemoteVolumeAsync(source.BlockVolume.RemoteFilename); // Re-inject the target if it has content if (target != null) { lst.Insert(lst.Count == 0 ? 0 : 1, target); } } foreach (var n in lst) { // We ignore the stop signal, but not the pause and terminate await taskreader.ProgressAsync; n.BlockVolume.Close(); await self.Output.WriteAsync(n); } })); }