public MeasurementToEventMessageAsyncCollectorTests()
{
_eventHubService = Substitute.For <IEventHubMessageService>();
_hashCodeFactory = Substitute.For <IHashCodeFactory>();
_telemetryLogger = Substitute.For <ITelemetryLogger>();
_measurementCollector = new MeasurementToEventMessageAsyncCollector(_eventHubService, _hashCodeFactory, _telemetryLogger);
_hashCodeGenerator = Substitute.For <IHashCodeGenerator>();
_hashCodeGenerator.GenerateHashCode(Arg.Any <string>()).Returns("123");
_hashCodeFactory.CreateDeterministicHashCodeGenerator().Returns(_hashCodeGenerator);
}
public async Task AddAsync(IEnumerable <IMeasurement> items, CancellationToken cancellationToken = default)
{
EnsureArg.IsNotNull(items, nameof(items));
using (var hasher = _hashCodeFactory.CreateDeterministicHashCodeGenerator())
{
var submissionTasks = items
.GroupBy(m =>
{
// cast as byte to restrict to 256 possible values. This will lead to a greater change of measurements ending up in the same bucket,
// while providing partition keys with enough entropy for EventHub to better distribute them across partitions.
return(hasher.GenerateHashCode(m.DeviceId.ToLower()));
})
.Select(async grp =>
{
var partitionKey = grp.Key;
var currentEventDataBatch = await _eventHubService.CreateEventDataBatchAsync(partitionKey);
foreach (var m in grp)
{
var measurementContent = JsonConvert.SerializeObject(m, Formatting.None);
var contentBytes = Encoding.UTF8.GetBytes(measurementContent);
var eventData = new EventData(contentBytes);
if (!currentEventDataBatch.TryAdd(eventData))
{
// The current EventDataBatch cannot hold any more events. Create a new EventDataBatch and add this new message to it.
var newEventDataBatch = await _eventHubService.CreateEventDataBatchAsync(partitionKey);
if (!newEventDataBatch.TryAdd(eventData))
{
// The measurement event is greater than the size allowed by EventHub. Log and discard. Keep the existing batch as there may
// be room for more events.
// TODO in this case we should send this to a dead letter queue. We'd need to see how we can send it, as it is too big for EventHub...
_telemetryLogger.LogError(new ArgumentOutOfRangeException($"A measurement event exceeded the maximum message batch size of {newEventDataBatch.MaximumSizeInBytes} bytes. It will be skipped."));
}
else
{
// Submit the current batch, and replace the currentEventDataBatch with newEventDataBatch
await _eventHubService.SendAsync(currentEventDataBatch, cancellationToken);
currentEventDataBatch.Dispose();
currentEventDataBatch = newEventDataBatch;
}
}
}
// Send over the remaining events
await _eventHubService.SendAsync(currentEventDataBatch, cancellationToken);
});
await Task.WhenAll(submissionTasks);
}
}