Esempio n. 1
0
        public static async Task <TResult> CopyAccountAsync <TResult>(this BlobCopyOptions options, CloudStorageAccount sourceAccount, CloudStorageAccount targetAccount, Func <bool> stopCalled, Func <string[], TResult> onCompleted)
        {
            var sourceClient = sourceAccount.CreateCloudBlobClient();
            var targetClient = targetAccount.CreateCloudBlobClient();

            return(onCompleted(await await sourceClient.FindAllContainersAsync(stopCalled,
                                                                               async sourceContainers =>
            {
                var stats = await sourceContainers
                            .Where(sourceContainer => options.ShouldCopy(sourceContainer.Name))
                            .Select(sourceContainer => sourceContainer.CopyContainerAsync(targetClient, options, stopCalled))
                            .WhenAllAsync(options.maxContainerConcurrency);
                return stats
                .SelectMany(s => s.Value.errors)
                .ToArray();
            },
                                                                               why => new[] { why }.ToTask())));
        }
Esempio n. 2
0
        private static async Task <KeyValuePair <string, BlobTransferStatistics> > CopyContainerAsync(this CloudBlobContainer sourceContainer, CloudBlobClient targetClient, BlobCopyOptions copyOptions, Func <bool> stopCalled)
        {
            var targetContainerName = sourceContainer.Name;

            if (stopCalled())
            {
                return(sourceContainer.Name.PairWithValue(BlobTransferStatistics.Default.Concat(new[] { $"copy stopped on {targetContainerName}" })));
            }

            return(await await sourceContainer.CreateIfNotExistTargetContainerForCopyAsync(targetClient, targetContainerName,
                                                                                           async (targetContainer, findExistingAsync, renewAccessAsync, releaseAccessAsync) =>
            {
                EastFiveAzureStorageBackupService.Log.Info($"starting {targetContainerName}");
                try
                {
                    var existingTargetBlobs = await findExistingAsync(stopCalled);
                    EastFiveAzureStorageBackupService.Log.Info($"{existingTargetBlobs.Value.Count} blobs already backed up for {targetContainerName}");
                    var pair = default(BlobContinuationToken).PairWithValue(BlobTransferStatistics.Default.Concat(existingTargetBlobs.Key));
                    BlobAccess access = default(BlobAccess);
                    Func <Task <BlobAccess> > renewWhenExpiredAsync =
                        async() =>
                    {
                        if (access.expiresUtc < DateTime.UtcNow + TimeSpan.FromMinutes(5))
                        {
                            access = await renewAccessAsync(copyOptions.accessPeriod);
                            await Task.Delay(TimeSpan.FromSeconds(10));          // let settle in so first copy will be ok
                        }
                        return access;
                    };
                    while (true)
                    {
                        if (stopCalled())
                        {
                            return sourceContainer.Name.PairWithValue(pair.Value.Concat(new[] { $"copy stopped on {sourceContainer.Name}" }));
                        }

                        pair = await await sourceContainer.FindNextBlobSegmentAsync <Task <KeyValuePair <BlobContinuationToken, BlobTransferStatistics> > >(pair.Key,
                                                                                                                                                            async(token, blobs) =>
                        {
                            var stats = await blobs.CopyBlobsWithContainerKeyAsync(targetContainer, existingTargetBlobs.Value,
                                                                                   () => pair.Value.calc.GetNextInterval(copyOptions.minIntervalCheckCopy, copyOptions.maxIntervalCheckCopy),
                                                                                   copyOptions.maxTotalWaitForCopy, renewWhenExpiredAsync, copyOptions.maxBlobConcurrencyPerContainer, stopCalled);
                            blobs = null;
                            return token.PairWithValue(pair.Value.Concat(stats));
                        },
                                                                                                                                                            why => default(BlobContinuationToken).PairWithValue(pair.Value.Concat(new[] { why })).ToTask());
                        pair.Value.LogProgress(copyOptions.minIntervalCheckCopy, copyOptions.maxIntervalCheckCopy,
                                               msg => EastFiveAzureStorageBackupService.Log.Info($"(progress) {targetContainerName} -> {msg}"));

                        if (default(BlobContinuationToken) == pair.Key)
                        {
                            if (pair.Value.retries.Any())
                            {
                                EastFiveAzureStorageBackupService.Log.Info($"retrying {targetContainerName}");
                                var copyRetries = copyOptions.copyRetries;
                                existingTargetBlobs = await findExistingAsync(stopCalled);
                                EastFiveAzureStorageBackupService.Log.Info($"{existingTargetBlobs.Value.Count} blobs already backed up for {targetContainerName}");
                                pair = default(BlobContinuationToken).PairWithValue(pair.Value.Concat(existingTargetBlobs.Key));     // just copies errors
                                var checkCopyCompleteAfter = pair.Value.calc.GetNextInterval(copyOptions.minIntervalCheckCopy, copyOptions.maxIntervalCheckCopy);
                                while (copyRetries-- > 0)
                                {
                                    if (stopCalled())
                                    {
                                        pair = default(BlobContinuationToken).PairWithValue(pair.Value.Concat(new[] { $"copy stopped on {sourceContainer.Name}" }));
                                        break;
                                    }
                                    var stats = await pair.Value.retries
                                                .Select(x => x.Key)
                                                .ToArray()
                                                .CopyBlobsWithContainerKeyAsync(targetContainer, existingTargetBlobs.Value,
                                                                                () => pair.Value.calc.GetNextInterval(copyOptions.minIntervalCheckCopy, copyOptions.maxIntervalCheckCopy),
                                                                                copyOptions.maxTotalWaitForCopy, renewWhenExpiredAsync, copyOptions.maxBlobConcurrencyPerContainer, stopCalled);
                                    pair = default(BlobContinuationToken).PairWithValue(new BlobTransferStatistics
                                    {
                                        calc = pair.Value.calc.Concat(stats.calc),
                                        errors = pair.Value.errors.Concat(stats.errors).ToArray(),
                                        successes = pair.Value.successes + stats.successes,
                                        retries = stats.retries,
                                        failures = pair.Value.failures.Concat(stats.failures).ToArray()
                                    });
                                    if (!pair.Value.retries.Any())
                                    {
                                        break;
                                    }
                                }
                            }
                            pair.Value.LogProgress(copyOptions.minIntervalCheckCopy, copyOptions.maxIntervalCheckCopy,
                                                   msg => EastFiveAzureStorageBackupService.Log.Info($"finished {targetContainerName} -> {msg}"),
                                                   BlobTransferStatistics.logFrequency);
                            return sourceContainer.Name.PairWithValue(pair.Value);
                        }
                    }
                }
                catch (Exception e)
                {
                    return sourceContainer.Name.PairWithValue(BlobTransferStatistics.Default.Concat(new [] { $"Exception copying container, Detail: {e.Message}" }));
                }
                finally
                {
                    await releaseAccessAsync();
                    EastFiveAzureStorageBackupService.Log.Info($"done with {targetContainerName}");
                }
            },
                                                                                           why => sourceContainer.Name.PairWithValue(BlobTransferStatistics.Default.Concat(new[] { why })).ToTask()));
        }