public static async Task <TResult> CopyAccountAsync <TResult>(this BlobCopyOptions options, CloudStorageAccount sourceAccount, CloudStorageAccount targetAccount, Func <bool> stopCalled, Func <string[], TResult> onCompleted)
{
var sourceClient = sourceAccount.CreateCloudBlobClient();
var targetClient = targetAccount.CreateCloudBlobClient();
return(onCompleted(await await sourceClient.FindAllContainersAsync(stopCalled,
async sourceContainers =>
{
var stats = await sourceContainers
.Where(sourceContainer => options.ShouldCopy(sourceContainer.Name))
.Select(sourceContainer => sourceContainer.CopyContainerAsync(targetClient, options, stopCalled))
.WhenAllAsync(options.maxContainerConcurrency);
return stats
.SelectMany(s => s.Value.errors)
.ToArray();
},
why => new[] { why }.ToTask())));
}
private static async Task <KeyValuePair <string, BlobTransferStatistics> > CopyContainerAsync(this CloudBlobContainer sourceContainer, CloudBlobClient targetClient, BlobCopyOptions copyOptions, Func <bool> stopCalled)
{
var targetContainerName = sourceContainer.Name;
if (stopCalled())
{
return(sourceContainer.Name.PairWithValue(BlobTransferStatistics.Default.Concat(new[] { $"copy stopped on {targetContainerName}" })));
}
return(await await sourceContainer.CreateIfNotExistTargetContainerForCopyAsync(targetClient, targetContainerName,
async (targetContainer, findExistingAsync, renewAccessAsync, releaseAccessAsync) =>
{
EastFiveAzureStorageBackupService.Log.Info($"starting {targetContainerName}");
try
{
var existingTargetBlobs = await findExistingAsync(stopCalled);
EastFiveAzureStorageBackupService.Log.Info($"{existingTargetBlobs.Value.Count} blobs already backed up for {targetContainerName}");
var pair = default(BlobContinuationToken).PairWithValue(BlobTransferStatistics.Default.Concat(existingTargetBlobs.Key));
BlobAccess access = default(BlobAccess);
Func <Task <BlobAccess> > renewWhenExpiredAsync =
async() =>
{
if (access.expiresUtc < DateTime.UtcNow + TimeSpan.FromMinutes(5))
{
access = await renewAccessAsync(copyOptions.accessPeriod);
await Task.Delay(TimeSpan.FromSeconds(10)); // let settle in so first copy will be ok
}
return access;
};
while (true)
{
if (stopCalled())
{
return sourceContainer.Name.PairWithValue(pair.Value.Concat(new[] { $"copy stopped on {sourceContainer.Name}" }));
}
pair = await await sourceContainer.FindNextBlobSegmentAsync <Task <KeyValuePair <BlobContinuationToken, BlobTransferStatistics> > >(pair.Key,
async(token, blobs) =>
{
var stats = await blobs.CopyBlobsWithContainerKeyAsync(targetContainer, existingTargetBlobs.Value,
() => pair.Value.calc.GetNextInterval(copyOptions.minIntervalCheckCopy, copyOptions.maxIntervalCheckCopy),
copyOptions.maxTotalWaitForCopy, renewWhenExpiredAsync, copyOptions.maxBlobConcurrencyPerContainer, stopCalled);
blobs = null;
return token.PairWithValue(pair.Value.Concat(stats));
},
why => default(BlobContinuationToken).PairWithValue(pair.Value.Concat(new[] { why })).ToTask());
pair.Value.LogProgress(copyOptions.minIntervalCheckCopy, copyOptions.maxIntervalCheckCopy,
msg => EastFiveAzureStorageBackupService.Log.Info($"(progress) {targetContainerName} -> {msg}"));
if (default(BlobContinuationToken) == pair.Key)
{
if (pair.Value.retries.Any())
{
EastFiveAzureStorageBackupService.Log.Info($"retrying {targetContainerName}");
var copyRetries = copyOptions.copyRetries;
existingTargetBlobs = await findExistingAsync(stopCalled);
EastFiveAzureStorageBackupService.Log.Info($"{existingTargetBlobs.Value.Count} blobs already backed up for {targetContainerName}");
pair = default(BlobContinuationToken).PairWithValue(pair.Value.Concat(existingTargetBlobs.Key)); // just copies errors
var checkCopyCompleteAfter = pair.Value.calc.GetNextInterval(copyOptions.minIntervalCheckCopy, copyOptions.maxIntervalCheckCopy);
while (copyRetries-- > 0)
{
if (stopCalled())
{
pair = default(BlobContinuationToken).PairWithValue(pair.Value.Concat(new[] { $"copy stopped on {sourceContainer.Name}" }));
break;
}
var stats = await pair.Value.retries
.Select(x => x.Key)
.ToArray()
.CopyBlobsWithContainerKeyAsync(targetContainer, existingTargetBlobs.Value,
() => pair.Value.calc.GetNextInterval(copyOptions.minIntervalCheckCopy, copyOptions.maxIntervalCheckCopy),
copyOptions.maxTotalWaitForCopy, renewWhenExpiredAsync, copyOptions.maxBlobConcurrencyPerContainer, stopCalled);
pair = default(BlobContinuationToken).PairWithValue(new BlobTransferStatistics
{
calc = pair.Value.calc.Concat(stats.calc),
errors = pair.Value.errors.Concat(stats.errors).ToArray(),
successes = pair.Value.successes + stats.successes,
retries = stats.retries,
failures = pair.Value.failures.Concat(stats.failures).ToArray()
});
if (!pair.Value.retries.Any())
{
break;
}
}
}
pair.Value.LogProgress(copyOptions.minIntervalCheckCopy, copyOptions.maxIntervalCheckCopy,
msg => EastFiveAzureStorageBackupService.Log.Info($"finished {targetContainerName} -> {msg}"),
BlobTransferStatistics.logFrequency);
return sourceContainer.Name.PairWithValue(pair.Value);
}
}
}
catch (Exception e)
{
return sourceContainer.Name.PairWithValue(BlobTransferStatistics.Default.Concat(new [] { $"Exception copying container, Detail: {e.Message}" }));
}
finally
{
await releaseAccessAsync();
EastFiveAzureStorageBackupService.Log.Info($"done with {targetContainerName}");
}
},
why => sourceContainer.Name.PairWithValue(BlobTransferStatistics.Default.Concat(new[] { why })).ToTask()));
}