private static async Task <TResult> CreateIfNotExistTargetTableForCopyAsync <TResult>(this CloudTable sourceTable,
CloudTableClient targetClient, string targetTableName, string[] partitionKeys, int numberOfSegments,
Func <CloudTable, Func <Func <bool>, Task <KeyValuePair <string[], IDictionary <string, SparseEntity> > > >, Func <SegmentedQuery[], Func <bool>, Task <KeyValuePair <string[], SegmentedQueryResult> > >, TResult> onSuccess,
Func <string, TResult> onFailure)
{
try
{
var targetTable = targetClient.GetTableReference(targetTableName);
var context = new OperationContext();
var exists = await targetTable.ExistsAsync(TableCopyOptions.requestOptions, context);
if (!exists)
{
await targetTable.CreateAsync(TableCopyOptions.requestOptions, context);
var createPermissions = await sourceTable.GetPermissionsAsync(TableCopyOptions.requestOptions, context);
await targetTable.SetPermissionsAsync(createPermissions, TableCopyOptions.requestOptions, context);
}
return(onSuccess(targetTable,
(stopCalled) =>
{
if (partitionKeys != null && partitionKeys.Length > 0)
{
return targetTable.FindAllRowsByPartitionKeysAsync(partitionKeys, stopCalled,
rows => new string[] { }.PairWithValue(rows),
(why, partialRowList) => new[] { why }.PairWithValue(partialRowList));
}
else
{
return targetTable.FindAllRowsByQueryAsync(
new TableQuery <DynamicTableEntity>().Select(new string[] { "PartitionKey" }), // we are not interested in the properties of the target so don't download them
stopCalled,
rows => new string[] { }.PairWithValue((IDictionary <string, SparseEntity>)rows.ToDictionary()),
(why, partialRowList) => new[] { why }.PairWithValue((IDictionary <string, SparseEntity>)partialRowList.ToDictionary()));
}
},
(details, stopCalled) =>
{
if (partitionKeys != null && partitionKeys.Length > 0)
{
var innerDetails = details.Any() ? details : SegmentedQuery.GetForPartitionKeys(partitionKeys); // init the run
return sourceTable.FindNextTableSegmentByPartitionKeysAsync(innerDetails, numberOfSegments, stopCalled,
result => new string[] { }.PairWithValue(result),
(why, result) => new[] { why }.PairWithValue(result));
}
else
{
var innerDetails = details.Any() ? details.First() : SegmentedQuery.Default; // init the run
return sourceTable.FindNextTableSegmentByQueryAsync(innerDetails, numberOfSegments, stopCalled,
(result) => new string[] { }.PairWithValue(result),
(why, result) => new[] { why }.PairWithValue(result));
}
}));
}
catch (Exception e)
{
return(onFailure($"Exception preparing table for copy, Detail: {e.Message}"));
}
}
private static async Task <TResult> FindNextTableSegmentByQueryAsync <TResult>(this CloudTable sourceTable, SegmentedQuery detail, int numberOfSegments, Func <bool> stopCalled,
Func <SegmentedQueryResult, TResult> onSuccess, Func <string, SegmentedQueryResult, TResult> onFailure)
{
var context = new OperationContext();
var list = new List <DynamicTableEntity>(numberOfSegments * 1000);
var retryTimes = ServiceSettings.defaultMaxAttempts;
while (true)
{
try
{
if (stopCalled())
{
return(onFailure($"stopped finding segments for {sourceTable.Name}", new SegmentedQueryResult
{
details = new SegmentedQuery[] { },
rows = list.ToArray()
}));
}
var segment = await sourceTable.ExecuteQuerySegmentedAsync(detail.query, detail.token, TableCopyOptions.requestOptions, context);
list.AddRange(segment.Results);
detail = detail.UpdateToken(segment.ContinuationToken);
if (detail.token == null)
{
return(onSuccess(new SegmentedQueryResult
{
details = new SegmentedQuery[] { },
rows = list.ToArray()
}));
}
if (--numberOfSegments < 1)
{
return(onSuccess(new SegmentedQueryResult
{
details = new[] { detail },
rows = list.ToArray()
}));
}
}
catch (Exception e)
{
if (e.Message.Contains("could not finish the operation within specified timeout") && --retryTimes > 0)
{
await Task.Delay(ServiceSettings.defaultBackoff);
continue;
}
return(onFailure($"Exception retrieving next table segment, Detail: {e.Message}", new SegmentedQueryResult
{
details = new SegmentedQuery[] { },
rows = list.ToArray()
}));
}
}
}
private static async Task <KeyValuePair <string, TableTransferStatistics> > CopyTableAsync(this CloudTable sourceTable, CloudTableClient targetClient, TableCopyOptions copyOptions, Func <bool> stopCalled)
{
var targetTableName = sourceTable.Name;
if (stopCalled())
{
return(sourceTable.Name.PairWithValue(TableTransferStatistics.Default.Concat(new[] { $"copy stopped on {targetTableName}" })));
}
return(await await sourceTable.CreateIfNotExistTargetTableForCopyAsync(targetClient, targetTableName, copyOptions.partitionKeys, copyOptions.maxSegmentDownloadConcurrencyPerTable,
async (targetTable, findExistingAsync, getNextSegmentAsync) =>
{
EastFiveAzureStorageBackupService.Log.Info($"starting {targetTableName}");
try
{
var existingTargetRows = await findExistingAsync(stopCalled);
EastFiveAzureStorageBackupService.Log.Info($"{existingTargetRows.Value.Count} entities already backed up for {targetTableName}");
var pair = new SegmentedQuery[] { }.PairWithValue(TableTransferStatistics.Default.Concat(existingTargetRows.Key)); // just copies errors
while (true)
{
if (stopCalled())
{
return sourceTable.Name.PairWithValue(pair.Value);
}
var nextSourceRows = await getNextSegmentAsync(pair.Key, stopCalled);
pair = nextSourceRows.Value.details
.PairWithValue(pair.Value.Concat(nextSourceRows.Key) // just copies errors
.Concat(await nextSourceRows.Value.rows.CopyRowsAsync(targetTable, existingTargetRows.Value, copyOptions.maxRowUploadConcurrencyPerTable))); // gets result of work
pair.Value.LogProgress(
msg => EastFiveAzureStorageBackupService.Log.Info($"(progress) {targetTableName} -> {msg}"));
if (!pair.Key.Any()) // wait until all the tokens are gone
{
if (pair.Value.retries.Any())
{
EastFiveAzureStorageBackupService.Log.Info($"retrying {targetTableName}");
var copyRetries = copyOptions.copyRetries;
existingTargetRows = await findExistingAsync(stopCalled);
EastFiveAzureStorageBackupService.Log.Info($"{existingTargetRows.Value.Count} entities already backed up for {targetTableName}");
pair = new SegmentedQuery[] { }.PairWithValue(pair.Value.Concat(existingTargetRows.Key)); // just copies errors
while (copyRetries-- > 0)
{
if (stopCalled())
{
break;
}
var stats = await pair.Value.retries
.Select(x => x.Key)
.ToArray()
.CopyRowsAsync(targetTable, existingTargetRows.Value, copyOptions.maxRowUploadConcurrencyPerTable);
pair = new SegmentedQuery[] { }.PairWithValue(new TableTransferStatistics
{
errors = pair.Value.errors.Concat(stats.errors).ToArray(),
successes = pair.Value.successes + stats.successes,
retries = stats.retries
});
if (!pair.Value.retries.Any())
{
break;
}
}
}
pair.Value.LogProgress(
msg => EastFiveAzureStorageBackupService.Log.Info($"finished {targetTableName} -> {msg}"),
TableTransferStatistics.logFrequency);
return sourceTable.Name.PairWithValue(pair.Value);
}
}
}
catch (Exception e)
{
return sourceTable.Name.PairWithValue(TableTransferStatistics.Default.Concat(new[] { $"Exception copying table, Detail: {e.Message}" }));
}
},
why => sourceTable.Name.PairWithValue(TableTransferStatistics.Default.Concat(new[] { why })).ToTask()));
}