public async Task GivenEvents_WhenProcessAsync_ThenIngestionTimeUtcSet_Test()
{
_template.GetMeasurements(null)
.ReturnsForAnyArgs(
new[] { new Measurement() },
new[] { new Measurement() },
new[] { new Measurement() },
new[] { new Measurement() },
new[] { new Measurement() },
new[] { new Measurement() },
new[] { new Measurement() },
new[] { new Measurement() },
new[] { new Measurement() },
new[] { new Measurement() });
var events = Enumerable.Range(0, 10).Select(i => BuildEvent(i)).ToArray();
var srv = new MeasurementEventNormalizationService(_logger, _template, _converter, _exceptionTelemetryProcessor, 1);
await srv.ProcessAsync(events, _consumer);
_template.ReceivedWithAnyArgs(10).GetMeasurements(null);
_converter.ReceivedWithAnyArgs(10).Convert(null);
await _consumer.Received(1).AddAsync(
Arg.Is <IEnumerable <IMeasurement> >(
measurements =>
measurements.Count() == 10 &&
measurements.Where(m => events.Any(e => m.IngestionTimeUtc == e.SystemProperties.EnqueuedTimeUtc)).Count() == 10),
Arg.Any <CancellationToken>());
}
public async Task GivenEventWithTwoResultPer_WhenProcessAsync_ThenEachEventResultConsumedTwice_Test()
{
var events = Enumerable.Range(0, 10)
.Select(i => BuildEvent(i))
.ToDictionary(ed => ed, ed => JToken.FromObject(new object()));
var template = Substitute.For <IContentTemplate>();
template.GetMeasurements(null).ReturnsForAnyArgs(new[] { Substitute.For <Measurement>(), Substitute.For <Measurement>() });
var converter = Substitute.For <Data.IConverter <EventData, JToken> >();
converter.Convert(null).ReturnsForAnyArgs(args => events[args.Arg <EventData>()]);
var log = Substitute.For <ITelemetryLogger>();
var consumer = Substitute.For <IAsyncCollector <IMeasurement> >();
var srv = new MeasurementEventNormalizationService(log, template, converter, 3);
await srv.ProcessAsync(events.Keys, consumer);
template.ReceivedWithAnyArgs(events.Count).GetMeasurements(null);
converter.ReceivedWithAnyArgs(events.Count).Convert(null);
await consumer.ReceivedWithAnyArgs(events.Count * 2).AddAsync(null);
foreach (var evt in events)
{
converter.Received(1).Convert(evt.Key);
template.Received(1).GetMeasurements(evt.Value);
}
}
public static async Task NormalizeDeviceData(
[EventHubTrigger("input", Connection = "InputEventHub")] EventData[] events,
[EventHubMeasurementCollector("output", Connection = "OutputEventHub")] IAsyncCollector <IMeasurement> output,
[Blob("template/%Template:DeviceContent%", FileAccess.Read)] string templateDefinitions,
ILogger log)
{
try
{
EnsureArg.IsNotNull(templateDefinitions, nameof(templateDefinitions));
EnsureArg.IsNotNull(events, nameof(events));
var templateContext = CollectionContentTemplateFactory.Default.Create(templateDefinitions);
templateContext.EnsureValid();
var template = templateContext.Template;
log.LogMetric(Metrics.DeviceEvent, events.Length);
IDataNormalizationService <EventData, IMeasurement> dataNormalizationService = new MeasurementEventNormalizationService(log, template);
await dataNormalizationService.ProcessAsync(events, output).ConfigureAwait(false);
}
catch (Exception ex)
{
log.RecordUnhandledExceptionMetrics(ex, nameof(NormalizeDeviceData));
throw;
}
}
public async Task GivenEventsAndNormalization_WhenProcessAsyncAndDataMappingErrors_ThenExecutionNotHaltedAndNormalizationDataMappingExceptionThrown_Test()
{
_template.GetMeasurements(null).ReturnsForAnyArgs(m => throw new Exception("Test"));
var events = Enumerable.Range(0, 10)
.Select(i => BuildEvent(i))
.ToDictionary(ed => ed, ed => JToken.FromObject(new object()));
_converter.Convert(null).ReturnsForAnyArgs(args => events[args.Arg <EventData>()]);
_exceptionTelemetryProcessor = new NormalizationExceptionTelemetryProcessor();
var srv = new MeasurementEventNormalizationService(_logger, _template, _converter, _exceptionTelemetryProcessor, 1);
var exception = await Assert.ThrowsAsync <AggregateException>(() => srv.ProcessAsync(events.Keys, _consumer));
Action <Exception> validateInnerExceptions = e =>
{
Assert.IsType <NormalizationDataMappingException>(e);
Assert.Equal("Test", e.Message);
};
// Verify all exceptions in the returned aggregate exception
Assert.Collection(exception.InnerExceptions, Enumerable.Range(0, 10).Select(i => validateInnerExceptions).ToArray());
_template.ReceivedWithAnyArgs(10).GetMeasurements(null);
_converter.ReceivedWithAnyArgs(10).Convert(null);
await _consumer.ReceivedWithAnyArgs(0).AddAsync(null);
}
public async Task GivenMultipleEventsWithOneResultPer_WhenProcessAsync_ThenEachEventConsumed_Test()
{
_template.GetMeasurements(null).ReturnsForAnyArgs(new[] { Substitute.For <Measurement>() });
var events = Enumerable.Range(0, 10).Select(i => BuildEvent(i)).ToArray();
var srv = new MeasurementEventNormalizationService(_logger, _template, _exceptionTelemetryProcessor);
await srv.ProcessAsync(events, _consumer);
_template.ReceivedWithAnyArgs(events.Length).GetMeasurements(null);
await _consumer.Received(1).AddAsync(Arg.Is <IEnumerable <IMeasurement> >(l => l.Count() == 10), Arg.Any <CancellationToken>());
}
public async Task GivenMultipleEventsWithOneResultPer_WhenTotalMeauresmentsProcessed_AreGreaterThanBatchSize_ThenEachEventConsumedInMultipleBatches_Test()
{
_template.GetMeasurements(null).ReturnsForAnyArgs(new[] { Substitute.For <Measurement>() });
var events = Enumerable.Range(0, 51).Select(i => BuildEvent(i)).ToArray();
var srv = new MeasurementEventNormalizationService(_logger, _template, _converter, _exceptionTelemetryProcessor, 3, 25);
await srv.ProcessAsync(events, _consumer);
_template.ReceivedWithAnyArgs(events.Length).GetMeasurements(null);
// 51 Events with a batch size of 25 = 2 * 25 measurement batches and 1 * 1 measurement batch
await _consumer.Received(2).AddAsync(Arg.Is <IEnumerable <IMeasurement> >(l => l.Count() == 25), Arg.Any <CancellationToken>());
await _consumer.Received(1).AddAsync(Arg.Is <IEnumerable <IMeasurement> >(l => l.Count() == 1), Arg.Any <CancellationToken>());
}
public async Task GivenEventsAndTaskCancellationException_WhenProcessAsync_ThenExecutionHalted_Test()
{
_template.GetMeasurements(null).ReturnsForAnyArgs(new[] { Substitute.For <Measurement>(), Substitute.For <Measurement>() });
var events = Enumerable.Range(0, 10).Select(i => BuildEvent(i)).ToArray();
_consumer.AddAsync(null).ReturnsForAnyArgs(v => Task.FromException(new TaskCanceledException()));
var srv = new MeasurementEventNormalizationService(_logger, _template, _converter, _exceptionTelemetryProcessor, 1);
var exception = await Assert.ThrowsAsync <TaskCanceledException>(() => srv.ProcessAsync(events, _consumer));
_template.ReceivedWithAnyArgs(events.Length).GetMeasurements(null);
_converter.ReceivedWithAnyArgs(events.Length).Convert(null);
await _consumer.Received(1).AddAsync(Arg.Is <IEnumerable <IMeasurement> >(l => l.Count() == events.Length * 2), Arg.Any <CancellationToken>());
}
public async Task GivenEventsAndDefaultErrorConsumer_WhenProcessAsyncAndHandleableConsumerErrors_ThenExceptionNotThrown_Test()
{
_template.GetMeasurements(null).ReturnsForAnyArgs(new[] { Substitute.For <Measurement>() });
var events = Enumerable.Range(0, 10).Select(i => BuildEvent(i)).ToArray();
_converter.Convert(null).ReturnsForAnyArgs(v => throw new IncompatibleDataException());
_exceptionTelemetryProcessor = new NormalizationExceptionTelemetryProcessor();
var srv = new MeasurementEventNormalizationService(_logger, _template, _converter, _exceptionTelemetryProcessor, 1);
await srv.ProcessAsync(events, _consumer);
_template.ReceivedWithAnyArgs(0).GetMeasurements(null);
_converter.ReceivedWithAnyArgs(events.Length).Convert(null);
await _consumer.ReceivedWithAnyArgs(0).AddAsync(null);
}
public async Task GivenEventsAndDefaultErrorConsumer_WhenProcessAsyncAndConsumerErrors_ThenEachEventResultConsumed_And_ErrorPerBatchProprogated_Test()
{
_template.GetMeasurements(null).ReturnsForAnyArgs(new[] { Substitute.For <Measurement>() });
var events = Enumerable.Range(0, 10).Select(i => BuildEvent(i)).ToArray();
_consumer.AddAsync(null).ReturnsForAnyArgs(v => Task.FromException(new Exception()));
var srv = new MeasurementEventNormalizationService(_logger, _template, _converter, _exceptionTelemetryProcessor, 1, 5); // Set asyncCollectorBatchSize to 5 to produce 2 batches
var exception = await Assert.ThrowsAsync <AggregateException>(() => srv.ProcessAsync(events, _consumer));
Assert.Equal(2, exception.InnerExceptions.Count);
_template.ReceivedWithAnyArgs(events.Length).GetMeasurements(null);
_converter.ReceivedWithAnyArgs(events.Length).Convert(null);
await _consumer.Received(2).AddAsync(Arg.Is <IEnumerable <IMeasurement> >(l => l.Count() == 5), Arg.Any <CancellationToken>());
}
public async Task GivenMultipleEventsWithOneResultPer_WhenProcessAsync_ThenEachEventConsumed_Test()
{
var template = Substitute.For <IContentTemplate>();
template.GetMeasurements(null).ReturnsForAnyArgs(new[] { Substitute.For <Measurement>() });
var events = Enumerable.Range(0, 10).Select(i => BuildEvent(i)).ToArray();
var log = Substitute.For <ITelemetryLogger>();
var consumer = Substitute.For <IAsyncCollector <IMeasurement> >();
var srv = new MeasurementEventNormalizationService(log, template);
await srv.ProcessAsync(events, consumer);
template.ReceivedWithAnyArgs(events.Length).GetMeasurements(null);
await consumer.ReceivedWithAnyArgs(events.Length).AddAsync(null);
}
public async Task NormalizeDeviceData(
[EventHubTrigger("input", Connection = "InputEventHub")] EventData[] events,
[EventHubMeasurementCollector("output", Connection = "OutputEventHub")] IAsyncCollector <IMeasurement> output,
[Blob("template/%Template:DeviceContent%", FileAccess.Read)] string templateDefinitions,
[DeviceDataNormalization] IOptions <NormalizationServiceOptions> normalizationSettings)
{
try
{
EnsureArg.IsNotNull(templateDefinitions, nameof(templateDefinitions));
EnsureArg.IsNotNull(events, nameof(events));
EnsureArg.IsNotNull(normalizationSettings, nameof(normalizationSettings));
var templateContext = _collectionContentTemplateFactory.Create(templateDefinitions);
templateContext.EnsureValid();
var template = templateContext.Template;
_logger.LogMetric(
IomtMetrics.DeviceEvent(),
events.Length);
IDataNormalizationService <EventData, IMeasurement> dataNormalizationService = new MeasurementEventNormalizationService(_logger, template, _exceptionTelemetryProcessor);
await dataNormalizationService.ProcessAsync(events, output).ConfigureAwait(false);
if (normalizationSettings.Value.LogDeviceIngressSizeBytes)
{
IEventProcessingMeter meter = new EventProcessingMeter();
var eventStats = await meter.CalculateEventStats(events);
_logger.LogMetric(
IomtMetrics.DeviceIngressSizeBytes(),
eventStats.TotalEventsProcessedBytes);
}
}
catch (Exception ex)
{
_logger.LogMetric(
IomtMetrics.UnhandledException(ex.GetType().Name, ConnectorOperation.Normalization),
1);
throw;
}
}
public async Task GivenEventWithTwoResultPer_WhenProcessAsync_ThenEachEventResultConsumedTwice_Test()
{
var events = Enumerable.Range(0, 10)
.Select(i => BuildEvent(i))
.ToDictionary(ed => ed, ed => JToken.FromObject(new object()));
_template.GetMeasurements(null).ReturnsForAnyArgs(new[] { Substitute.For <Measurement>(), Substitute.For <Measurement>() });
_converter.Convert(null).ReturnsForAnyArgs(args => events[args.Arg <EventData>()]);
var srv = new MeasurementEventNormalizationService(_logger, _template, _converter, _exceptionTelemetryProcessor, 3);
await srv.ProcessAsync(events.Keys, _consumer);
_template.ReceivedWithAnyArgs(events.Count).GetMeasurements(null);
_converter.ReceivedWithAnyArgs(events.Count).Convert(null);
await _consumer.Received(1).AddAsync(Arg.Is <IEnumerable <IMeasurement> >(l => l.Count() == events.Count * 2), Arg.Any <CancellationToken>());
foreach (var evt in events)
{
_converter.Received(1).Convert(evt.Key);
_template.Received(1).GetMeasurements(evt.Value);
}
}
public async Task GivenEventsAndTaskCancellationException_WhenProcessAsync_ThenExecutionHalted_Test()
{
var template = Substitute.For <IContentTemplate>();
template.GetMeasurements(null).ReturnsForAnyArgs(new[] { Substitute.For <Measurement>(), Substitute.For <Measurement>() });
var converter = Substitute.For <Data.IConverter <EventData, JToken> >();
var events = Enumerable.Range(0, 10).Select(i => BuildEvent(i)).ToArray();
var log = Substitute.For <ITelemetryLogger>();
var consumer = Substitute.For <IAsyncCollector <IMeasurement> >();
consumer.AddAsync(null).ReturnsForAnyArgs(v => Task.FromException(new TaskCanceledException()));
var srv = new MeasurementEventNormalizationService(log, template, converter, 1);
var exception = await Assert.ThrowsAsync <TaskCanceledException>(() => srv.ProcessAsync(events, consumer));
template.ReceivedWithAnyArgs(1).GetMeasurements(null);
converter.ReceivedWithAnyArgs(1).Convert(null);
await consumer.ReceivedWithAnyArgs(1).AddAsync(null);
}
public async Task GivenEvents_WhenProcessAsync_ThenIngestionTimeUtcSet_Test()
{
var template = Substitute.For <IContentTemplate>();
template.GetMeasurements(null)
.ReturnsForAnyArgs(
new[] { new Measurement() },
new[] { new Measurement() },
new[] { new Measurement() },
new[] { new Measurement() },
new[] { new Measurement() },
new[] { new Measurement() },
new[] { new Measurement() },
new[] { new Measurement() },
new[] { new Measurement() },
new[] { new Measurement() });
var converter = Substitute.For <Data.IConverter <EventData, JToken> >();
var events = Enumerable.Range(0, 10).Select(i => BuildEvent(i)).ToArray();
var log = Substitute.For <ITelemetryLogger>();
var consumer = Substitute.For <IAsyncCollector <IMeasurement> >();
var srv = new MeasurementEventNormalizationService(log, template, converter, 1);
await srv.ProcessAsync(events, consumer);
template.ReceivedWithAnyArgs(10).GetMeasurements(null);
converter.ReceivedWithAnyArgs(10).Convert(null);
await consumer.ReceivedWithAnyArgs(10).AddAsync(null);
foreach (var evt in events)
{
await consumer.Received(1)
.AddAsync(Arg.Is <Measurement>(m => m.IngestionTimeUtc == evt.SystemProperties.EnqueuedTimeUtc));
}
}
public async Task GivenEventsAndDefaultErrorConsumer_WhenProcessAsyncAndConsumerErrors_ThenEachEventResultConsumedAndErrorProprogated_Test()
{
var template = Substitute.For <IContentTemplate>();
template.GetMeasurements(null).ReturnsForAnyArgs(new[] { Substitute.For <Measurement>() });
var converter = Substitute.For <Data.IConverter <EventData, JToken> >();
var events = Enumerable.Range(0, 10).Select(i => BuildEvent(i)).ToArray();
var log = Substitute.For <ITelemetryLogger>();
var consumer = Substitute.For <IAsyncCollector <IMeasurement> >();
consumer.AddAsync(null).ReturnsForAnyArgs(v => Task.FromException(new Exception()));
var srv = new MeasurementEventNormalizationService(log, template, converter, 1);
var exception = await Assert.ThrowsAsync <AggregateException>(() => srv.ProcessAsync(events, consumer));
Assert.Equal(events.Length, exception.InnerExceptions.Count);
template.ReceivedWithAnyArgs(events.Length).GetMeasurements(null);
converter.ReceivedWithAnyArgs(events.Length).Convert(null);
await consumer.ReceivedWithAnyArgs(events.Length).AddAsync(null);
}