示例#1
0
        public static async Task <IndexVolumeWriter> CreateIndexVolume(string blockname, Options options, Common.DatabaseCommon database)
        {
            using (var h = Duplicati.Library.Utility.HashAlgorithmHelper.Create(options.BlockHashAlgorithm))
            {
                var w = new IndexVolumeWriter(options);
                w.VolumeID = await database.RegisterRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary);

                var blockvolume = await database.GetVolumeInfoAsync(blockname);

                w.StartVolume(blockname);
                foreach (var b in await database.GetBlocksAsync(blockvolume.ID))
                {
                    w.AddBlock(b.Hash, b.Size);
                }

                w.FinishVolume(blockvolume.Hash, blockvolume.Size);

                if (options.IndexfilePolicy == Options.IndexFileStrategy.Full)
                {
                    foreach (var b in await database.GetBlocklistsAsync(blockvolume.ID, options.Blocksize, options.BlockhashSize))
                    {
                        var bh = Convert.ToBase64String(h.ComputeHash(b.Item2, 0, b.Item3));
                        if (bh != b.Item1)
                        {
                            throw new Exception(string.Format("Internal consistency check failed, generated index block has wrong hash, {0} vs {1}", bh, b.Item1));
                        }
                        w.WriteBlocklist(b.Item1, b.Item2, 0, b.Item3);
                    }
                }

                w.Close();

                await database.AddIndexBlockLinkAsync(w.VolumeID, blockvolume.ID);

                return(w);
            }
        }
示例#2
0
        public static Task Run(Common.BackendHandler backend, Options options, Common.DatabaseCommon database, BackupResults results, Common.ITaskReader taskreader, StatsCollector stats)
        {
            return(AutomationExtensions.RunTask(new
            {
                Input = Channels.BackendRequest.ForRead,
            },

                                                async self =>
            {
                var inProgress = new Queue <KeyValuePair <int, Task> >();
                var max_pending = options.AsynchronousUploadLimit == 0 ? long.MaxValue : options.AsynchronousUploadLimit;
                var noIndexFiles = options.IndexfilePolicy == Options.IndexFileStrategy.None;
                var active = 0;
                var lastSize = -1L;

                while (!await self.Input.IsRetiredAsync && await taskreader.ProgressAsync)
                {
                    try
                    {
                        var req = await self.Input.ReadAsync();

                        if (!await taskreader.ProgressAsync)
                        {
                            continue;
                        }

                        var task = default(KeyValuePair <int, Task>);
                        if (req is VolumeUploadRequest)
                        {
                            lastSize = ((VolumeUploadRequest)req).BlockVolume.SourceSize;

                            if (noIndexFiles || ((VolumeUploadRequest)req).IndexVolume == null)
                            {
                                task = new KeyValuePair <int, Task>(1, backend.UploadFileAsync(((VolumeUploadRequest)req).BlockVolume, null));
                            }
                            else
                            {
                                task = new KeyValuePair <int, Task>(2, backend.UploadFileAsync(((VolumeUploadRequest)req).BlockVolume, name => ((VolumeUploadRequest)req).IndexVolume.CreateVolume(name, options, database)));
                            }
                        }
                        else if (req is FilesetUploadRequest)
                        {
                            task = new KeyValuePair <int, Task>(1, backend.UploadFileAsync(((FilesetUploadRequest)req).Fileset));
                        }
                        else if (req is IndexVolumeUploadRequest)
                        {
                            task = new KeyValuePair <int, Task>(1, backend.UploadFileAsync(((IndexVolumeUploadRequest)req).IndexVolume));
                        }
                        else if (req is FlushRequest)
                        {
                            try
                            {
                                while (inProgress.Count > 0)
                                {
                                    await inProgress.Dequeue().Value;
                                }
                                active = 0;
                            }
                            finally
                            {
                                ((FlushRequest)req).SetFlushed(lastSize);
                            }
                        }

                        if (task.Value != null)
                        {
                            inProgress.Enqueue(task);
                            active += task.Key;
                        }
                    }
                    catch (Exception ex)
                    {
                        if (!ex.IsRetiredException())
                        {
                            throw;
                        }
                    }

                    while (active >= max_pending)
                    {
                        var top = inProgress.Dequeue();

                        // See if we are done
                        if (await Task.WhenAny(top.Value, Task.Delay(500)) != top.Value)
                        {
                            try
                            {
                                stats.SetBlocking(true);
                                await top.Value;
                            }
                            finally
                            {
                                stats.SetBlocking(false);
                            }
                        }

                        active -= top.Key;
                    }
                }

                results.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload);

                try
                {
                    stats.SetBlocking(true);
                    while (inProgress.Count > 0)
                    {
                        await inProgress.Dequeue().Value;
                    }
                }
                finally
                {
                    stats.SetBlocking(false);
                }
            }));
        }
示例#3
0
        /// <summary>
        /// Creates an index volume with the temporary contents
        /// </summary>
        /// <returns>The index volume.</returns>
        /// <param name="blockfilename">The name of the block file.</param>
        public async Task <IndexVolumeWriter> CreateVolume(string blockfilename, Options options, Common.DatabaseCommon database)
        {
            var w = new IndexVolumeWriter(options);

            w.VolumeID = await database.RegisterRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary);

            var blockvolume = await database.GetVolumeInfoAsync(blockfilename);

            w.StartVolume(blockfilename);
            foreach (var n in blockHashes)
            {
                var args = n.Split(new char[] { ':' }, 2);
                w.AddBlock(args[1], long.Parse(args[0]));
            }

            w.FinishVolume(blockvolume.Hash, blockvolume.Size);

            var enumerator = blockListHashes.GetEnumerator();

            while (enumerator.MoveNext())
            {
                var hash = enumerator.Current;
                enumerator.MoveNext();
                var data = Convert.FromBase64String(enumerator.Current);

                w.WriteBlocklist(hash, data, 0, data.Length);
            }

            w.Close();

            return(w);
        }