public TableTransferStatistics Concat(TableTransferStatistics stats)
{
return(new TableTransferStatistics
{
errors = stats.errors.Any() ? this.errors.Concat(stats.errors).ToArray() : this.errors,
successes = this.successes + stats.successes,
retries = stats.retries.Any() ? this.retries.Concat(stats.retries).ToArray() : this.retries
});
}
private static async Task <TableTransferStatistics> CopyRowsAsync(this DynamicTableEntity[] sourceRows, CloudTable targetTable, IDictionary <string, SparseEntity> existingTargetRows, int maxRowUpload)
{
var toCopy = sourceRows
.Where(row => !existingTargetRows.TryGetValue(row.RowKey, out SparseEntity existing) ||
existing.timeStamp < row.Timestamp || existing.partitionKey != row.PartitionKey)
.ToArray();
var stats = new TableTransferStatistics
{
errors = new string[] { },
successes = sourceRows.Length - toCopy.Length,
retries = new KeyValuePair <DynamicTableEntity, TransferStatus>[] { }
};
if (toCopy.Length == 0)
{
return(stats);
}
return(await toCopy
.GroupBy(row => row.PartitionKey)
.SelectMany(group => group
.ToArray()
.Select((x, index) => new { x, index })
.GroupBy(x => x.index / maxRowUpload, y => y.x))
.Aggregate(stats.ToTask(),
async(aggrTask, group) =>
{
var innerResult = await group.ToArray().BatchInsertOrReplaceForSamePartitionKeyAsync(targetTable);
var innerStats = await aggrTask; // awaiting after insert to get more parallelism
return innerStats.Concat(new TableTransferStatistics
{
errors = innerResult.Key,
successes = innerResult.Value.Count(item => item.Value == TransferStatus.CopySuccessful),
retries = innerResult.Value.Where(item => item.Value == TransferStatus.ShouldRetry).ToArray()
});
}));
}