private static async Task <TResult> CreateIfNotExistTargetTableForCopyAsync <TResult>(this CloudTable sourceTable, CloudTableClient targetClient, string targetTableName, string[] partitionKeys, int numberOfSegments, Func <CloudTable, Func <Func <bool>, Task <KeyValuePair <string[], IDictionary <string, SparseEntity> > > >, Func <SegmentedQuery[], Func <bool>, Task <KeyValuePair <string[], SegmentedQueryResult> > >, TResult> onSuccess, Func <string, TResult> onFailure) { try { var targetTable = targetClient.GetTableReference(targetTableName); var context = new OperationContext(); var exists = await targetTable.ExistsAsync(TableCopyOptions.requestOptions, context); if (!exists) { await targetTable.CreateAsync(TableCopyOptions.requestOptions, context); var createPermissions = await sourceTable.GetPermissionsAsync(TableCopyOptions.requestOptions, context); await targetTable.SetPermissionsAsync(createPermissions, TableCopyOptions.requestOptions, context); } return(onSuccess(targetTable, (stopCalled) => { if (partitionKeys != null && partitionKeys.Length > 0) { return targetTable.FindAllRowsByPartitionKeysAsync(partitionKeys, stopCalled, rows => new string[] { }.PairWithValue(rows), (why, partialRowList) => new[] { why }.PairWithValue(partialRowList)); } else { return targetTable.FindAllRowsByQueryAsync( new TableQuery <DynamicTableEntity>().Select(new string[] { "PartitionKey" }), // we are not interested in the properties of the target so don't download them stopCalled, rows => new string[] { }.PairWithValue((IDictionary <string, SparseEntity>)rows.ToDictionary()), (why, partialRowList) => new[] { why }.PairWithValue((IDictionary <string, SparseEntity>)partialRowList.ToDictionary())); } }, (details, stopCalled) => { if (partitionKeys != null && partitionKeys.Length > 0) { var innerDetails = details.Any() ? details : SegmentedQuery.GetForPartitionKeys(partitionKeys); // init the run return sourceTable.FindNextTableSegmentByPartitionKeysAsync(innerDetails, numberOfSegments, stopCalled, result => new string[] { }.PairWithValue(result), (why, result) => new[] { why }.PairWithValue(result)); } else { var innerDetails = details.Any() ? details.First() : SegmentedQuery.Default; // init the run return sourceTable.FindNextTableSegmentByQueryAsync(innerDetails, numberOfSegments, stopCalled, (result) => new string[] { }.PairWithValue(result), (why, result) => new[] { why }.PairWithValue(result)); } })); } catch (Exception e) { return(onFailure($"Exception preparing table for copy, Detail: {e.Message}")); } }
private static async Task <TResult> FindNextTableSegmentByQueryAsync <TResult>(this CloudTable sourceTable, SegmentedQuery detail, int numberOfSegments, Func <bool> stopCalled, Func <SegmentedQueryResult, TResult> onSuccess, Func <string, SegmentedQueryResult, TResult> onFailure) { var context = new OperationContext(); var list = new List <DynamicTableEntity>(numberOfSegments * 1000); var retryTimes = ServiceSettings.defaultMaxAttempts; while (true) { try { if (stopCalled()) { return(onFailure($"stopped finding segments for {sourceTable.Name}", new SegmentedQueryResult { details = new SegmentedQuery[] { }, rows = list.ToArray() })); } var segment = await sourceTable.ExecuteQuerySegmentedAsync(detail.query, detail.token, TableCopyOptions.requestOptions, context); list.AddRange(segment.Results); detail = detail.UpdateToken(segment.ContinuationToken); if (detail.token == null) { return(onSuccess(new SegmentedQueryResult { details = new SegmentedQuery[] { }, rows = list.ToArray() })); } if (--numberOfSegments < 1) { return(onSuccess(new SegmentedQueryResult { details = new[] { detail }, rows = list.ToArray() })); } } catch (Exception e) { if (e.Message.Contains("could not finish the operation within specified timeout") && --retryTimes > 0) { await Task.Delay(ServiceSettings.defaultBackoff); continue; } return(onFailure($"Exception retrieving next table segment, Detail: {e.Message}", new SegmentedQueryResult { details = new SegmentedQuery[] { }, rows = list.ToArray() })); } } }
private static async Task <KeyValuePair <string, TableTransferStatistics> > CopyTableAsync(this CloudTable sourceTable, CloudTableClient targetClient, TableCopyOptions copyOptions, Func <bool> stopCalled) { var targetTableName = sourceTable.Name; if (stopCalled()) { return(sourceTable.Name.PairWithValue(TableTransferStatistics.Default.Concat(new[] { $"copy stopped on {targetTableName}" }))); } return(await await sourceTable.CreateIfNotExistTargetTableForCopyAsync(targetClient, targetTableName, copyOptions.partitionKeys, copyOptions.maxSegmentDownloadConcurrencyPerTable, async (targetTable, findExistingAsync, getNextSegmentAsync) => { EastFiveAzureStorageBackupService.Log.Info($"starting {targetTableName}"); try { var existingTargetRows = await findExistingAsync(stopCalled); EastFiveAzureStorageBackupService.Log.Info($"{existingTargetRows.Value.Count} entities already backed up for {targetTableName}"); var pair = new SegmentedQuery[] { }.PairWithValue(TableTransferStatistics.Default.Concat(existingTargetRows.Key)); // just copies errors while (true) { if (stopCalled()) { return sourceTable.Name.PairWithValue(pair.Value); } var nextSourceRows = await getNextSegmentAsync(pair.Key, stopCalled); pair = nextSourceRows.Value.details .PairWithValue(pair.Value.Concat(nextSourceRows.Key) // just copies errors .Concat(await nextSourceRows.Value.rows.CopyRowsAsync(targetTable, existingTargetRows.Value, copyOptions.maxRowUploadConcurrencyPerTable))); // gets result of work pair.Value.LogProgress( msg => EastFiveAzureStorageBackupService.Log.Info($"(progress) {targetTableName} -> {msg}")); if (!pair.Key.Any()) // wait until all the tokens are gone { if (pair.Value.retries.Any()) { EastFiveAzureStorageBackupService.Log.Info($"retrying {targetTableName}"); var copyRetries = copyOptions.copyRetries; existingTargetRows = await findExistingAsync(stopCalled); EastFiveAzureStorageBackupService.Log.Info($"{existingTargetRows.Value.Count} entities already backed up for {targetTableName}"); pair = new SegmentedQuery[] { }.PairWithValue(pair.Value.Concat(existingTargetRows.Key)); // just copies errors while (copyRetries-- > 0) { if (stopCalled()) { break; } var stats = await pair.Value.retries .Select(x => x.Key) .ToArray() .CopyRowsAsync(targetTable, existingTargetRows.Value, copyOptions.maxRowUploadConcurrencyPerTable); pair = new SegmentedQuery[] { }.PairWithValue(new TableTransferStatistics { errors = pair.Value.errors.Concat(stats.errors).ToArray(), successes = pair.Value.successes + stats.successes, retries = stats.retries }); if (!pair.Value.retries.Any()) { break; } } } pair.Value.LogProgress( msg => EastFiveAzureStorageBackupService.Log.Info($"finished {targetTableName} -> {msg}"), TableTransferStatistics.logFrequency); return sourceTable.Name.PairWithValue(pair.Value); } } } catch (Exception e) { return sourceTable.Name.PairWithValue(TableTransferStatistics.Default.Concat(new[] { $"Exception copying table, Detail: {e.Message}" })); } }, why => sourceTable.Name.PairWithValue(TableTransferStatistics.Default.Concat(new[] { why })).ToTask())); }