public static async Task <TResult> CopyAccountAsync <TResult>(this TableCopyOptions options, CloudStorageAccount sourceAccount, CloudStorageAccount targetAccount, Func <bool> stopCalled, Func <string[], TResult> onCompleted)
{
var sourceClient = sourceAccount.CreateCloudTableClient();
var targetClient = targetAccount.CreateCloudTableClient();
return(onCompleted(await await sourceClient.FindAllTablesAsync(stopCalled,
async sourceTables =>
{
var stats = await sourceTables
.Where(sourceTable => options.ShouldCopy(sourceTable.Name))
.Select(sourceTable => sourceTable.CopyTableAsync(targetClient, options, stopCalled))
.WhenAllAsync(options.maxTableConcurrency);
return stats
.SelectMany(s => s.Value.errors)
.ToArray();
},
why => new[] { why }.ToTask())));
}
private static async Task <KeyValuePair <string, TableTransferStatistics> > CopyTableAsync(this CloudTable sourceTable, CloudTableClient targetClient, TableCopyOptions copyOptions, Func <bool> stopCalled)
{
var targetTableName = sourceTable.Name;
if (stopCalled())
{
return(sourceTable.Name.PairWithValue(TableTransferStatistics.Default.Concat(new[] { $"copy stopped on {targetTableName}" })));
}
return(await await sourceTable.CreateIfNotExistTargetTableForCopyAsync(targetClient, targetTableName, copyOptions.partitionKeys, copyOptions.maxSegmentDownloadConcurrencyPerTable,
async (targetTable, findExistingAsync, getNextSegmentAsync) =>
{
EastFiveAzureStorageBackupService.Log.Info($"starting {targetTableName}");
try
{
var existingTargetRows = await findExistingAsync(stopCalled);
EastFiveAzureStorageBackupService.Log.Info($"{existingTargetRows.Value.Count} entities already backed up for {targetTableName}");
var pair = new SegmentedQuery[] { }.PairWithValue(TableTransferStatistics.Default.Concat(existingTargetRows.Key)); // just copies errors
while (true)
{
if (stopCalled())
{
return sourceTable.Name.PairWithValue(pair.Value);
}
var nextSourceRows = await getNextSegmentAsync(pair.Key, stopCalled);
pair = nextSourceRows.Value.details
.PairWithValue(pair.Value.Concat(nextSourceRows.Key) // just copies errors
.Concat(await nextSourceRows.Value.rows.CopyRowsAsync(targetTable, existingTargetRows.Value, copyOptions.maxRowUploadConcurrencyPerTable))); // gets result of work
pair.Value.LogProgress(
msg => EastFiveAzureStorageBackupService.Log.Info($"(progress) {targetTableName} -> {msg}"));
if (!pair.Key.Any()) // wait until all the tokens are gone
{
if (pair.Value.retries.Any())
{
EastFiveAzureStorageBackupService.Log.Info($"retrying {targetTableName}");
var copyRetries = copyOptions.copyRetries;
existingTargetRows = await findExistingAsync(stopCalled);
EastFiveAzureStorageBackupService.Log.Info($"{existingTargetRows.Value.Count} entities already backed up for {targetTableName}");
pair = new SegmentedQuery[] { }.PairWithValue(pair.Value.Concat(existingTargetRows.Key)); // just copies errors
while (copyRetries-- > 0)
{
if (stopCalled())
{
break;
}
var stats = await pair.Value.retries
.Select(x => x.Key)
.ToArray()
.CopyRowsAsync(targetTable, existingTargetRows.Value, copyOptions.maxRowUploadConcurrencyPerTable);
pair = new SegmentedQuery[] { }.PairWithValue(new TableTransferStatistics
{
errors = pair.Value.errors.Concat(stats.errors).ToArray(),
successes = pair.Value.successes + stats.successes,
retries = stats.retries
});
if (!pair.Value.retries.Any())
{
break;
}
}
}
pair.Value.LogProgress(
msg => EastFiveAzureStorageBackupService.Log.Info($"finished {targetTableName} -> {msg}"),
TableTransferStatistics.logFrequency);
return sourceTable.Name.PairWithValue(pair.Value);
}
}
}
catch (Exception e)
{
return sourceTable.Name.PairWithValue(TableTransferStatistics.Default.Concat(new[] { $"Exception copying table, Detail: {e.Message}" }));
}
},
why => sourceTable.Name.PairWithValue(TableTransferStatistics.Default.Concat(new[] { why })).ToTask()));
}
