private async Task InsertManyAsync <T, TState>(IAsyncEnumerable <DomainId> source, int batchSize, double errorThreshold,
CancellationToken ct = default)
where T : DomainObject <TState> where TState : class, IDomainState <TState>, new()
{
var store = serviceProvider.GetRequiredService <IStore <TState> >();
var parallelism = Environment.ProcessorCount;
var handledIds = new HashSet <DomainId>();
var handlerErrors = 0;
using (localCache.StartContext())
{
var workerBlock = new ActionBlock <DomainId[]>(async ids =>
{
try
{
await using (var context = store.WithBatchContext(typeof(T)))
{
await context.LoadAsync(ids);
foreach (var id in ids)
{
try
{
var domainObject = Factory <T, TState> .Create(serviceProvider, context);
domainObject.Setup(id);
await domainObject.RebuildStateAsync();
}
catch (DomainObjectNotFoundException)
{
return;
}
catch (Exception ex)
{
log.LogWarning(ex, w => w
.WriteProperty("reason", "CorruptData")
.WriteProperty("domainObjectId", id.ToString())
.WriteProperty("domainObjectType", typeof(T).Name));
Interlocked.Increment(ref handlerErrors);
}
}
}
}
catch (OperationCanceledException ex)
{
// Dataflow swallows operation cancelled exception.
throw new AggregateException(ex);
}
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = parallelism,
MaxMessagesPerTask = 10,
BoundedCapacity = parallelism
});
var batchBlock = new BatchBlock <DomainId>(batchSize, new GroupingDataflowBlockOptions
{
BoundedCapacity = batchSize
});
batchBlock.BidirectionalLinkTo(workerBlock);
await foreach (var id in source.WithCancellation(ct))
{
if (handledIds.Add(id))
{
if (!await batchBlock.SendAsync(id, ct))
{
break;
}
}
}
batchBlock.Complete();
await workerBlock.Completion;
}
var errorRate = (double)handlerErrors / handledIds.Count;
if (errorRate > errorThreshold)
{
throw new InvalidOperationException($"Error rate of {errorRate} is above threshold {errorThreshold}.");
}
}