public TableTransferStatistics Concat(TableTransferStatistics stats) { return(new TableTransferStatistics { errors = stats.errors.Any() ? this.errors.Concat(stats.errors).ToArray() : this.errors, successes = this.successes + stats.successes, retries = stats.retries.Any() ? this.retries.Concat(stats.retries).ToArray() : this.retries }); }
private static async Task <TableTransferStatistics> CopyRowsAsync(this DynamicTableEntity[] sourceRows, CloudTable targetTable, IDictionary <string, SparseEntity> existingTargetRows, int maxRowUpload) { var toCopy = sourceRows .Where(row => !existingTargetRows.TryGetValue(row.RowKey, out SparseEntity existing) || existing.timeStamp < row.Timestamp || existing.partitionKey != row.PartitionKey) .ToArray(); var stats = new TableTransferStatistics { errors = new string[] { }, successes = sourceRows.Length - toCopy.Length, retries = new KeyValuePair <DynamicTableEntity, TransferStatus>[] { } }; if (toCopy.Length == 0) { return(stats); } return(await toCopy .GroupBy(row => row.PartitionKey) .SelectMany(group => group .ToArray() .Select((x, index) => new { x, index }) .GroupBy(x => x.index / maxRowUpload, y => y.x)) .Aggregate(stats.ToTask(), async(aggrTask, group) => { var innerResult = await group.ToArray().BatchInsertOrReplaceForSamePartitionKeyAsync(targetTable); var innerStats = await aggrTask; // awaiting after insert to get more parallelism return innerStats.Concat(new TableTransferStatistics { errors = innerResult.Key, successes = innerResult.Value.Count(item => item.Value == TransferStatus.CopySuccessful), retries = innerResult.Value.Where(item => item.Value == TransferStatus.ShouldRetry).ToArray() }); })); }