public async Task GivenMultipleEventsWithOneResultPer_WhenProcessAsync_ThenEachEventConsumed_Test()
{
_template.GetMeasurements(null).ReturnsForAnyArgs(new[] { Substitute.For <Measurement>() });
var events = Enumerable.Range(0, 10).Select(i => BuildEvent(i)).ToArray();
var srv = new MeasurementEventNormalizationService(_logger, _template, _exceptionTelemetryProcessor);
await srv.ProcessAsync(events, _consumer);
_template.ReceivedWithAnyArgs(events.Length).GetMeasurements(null);
await _consumer.Received(1).AddAsync(Arg.Is <IEnumerable <IMeasurement> >(l => l.Count() == 10), Arg.Any <CancellationToken>());
}
public void GivenTemplateAndSingleMultiValueValidToken_WhenGetMeasurements_ThenSingleMeasurementReturned_Test(string eventJson)
{
var token = JsonConvert.DeserializeObject <JToken>(eventJson);
var result = MultiValueTemplate.GetMeasurements(token).ToArray();
Assert.NotNull(result);
Assert.Collection(result, m =>
{
Assert.Equal("bloodpressure", m.Type);
Assert.Equal(token["Properties"]["iothub-creation-time-utc"], m.OccurrenceTimeUtc);
Assert.Equal(token["SystemProperties"]["iothub-connection-device-id"], m.DeviceId);
Assert.Collection(
m.Properties,
p =>
{
Assert.Equal("systolic", p.Name);
Assert.Equal("111", p.Value);
},
p =>
{
Assert.Equal("diastolic", p.Name);
Assert.Equal("75", p.Value);
});
});
}
public void GivenBloodpressureTemplate_WhenGetMeasurements_ThenAllMeasurementReturned_Test(string eventJson)
{
var token = JsonConvert.DeserializeObject <JToken>(eventJson);
var result = BloodPressureTemplate.GetMeasurements(token).ToArray();
Assert.NotNull(result);
Assert.Collection(result, m =>
{
Assert.Equal("bloodPressure", m.Type);
Assert.Equal(token["enqueuedTime"], m.OccurrenceTimeUtc);
Assert.Equal(token["deviceId"], m.DeviceId);
Assert.Collection(
m.Properties,
p =>
{
Assert.Equal("bp_diastolic", p.Name);
Assert.Equal("7", p.Value);
},
p =>
{
Assert.Equal("bp_systolic", p.Name);
Assert.Equal("71", p.Value);
});
});
}
public void GivenMultiValueTemplateAndValidTokenWithAllValues_WhenGetMeasurements_ThenSingleMeasurementReturned_Test(IContentTemplate template)
{
var time = DateTime.UtcNow;
var token = JToken.FromObject(new { heartrate = "60", steps = "2", device = "abc", date = time });
var result = template.GetMeasurements(token).ToArray();
Assert.NotNull(result);
Assert.Collection(result, m =>
{
Assert.Equal("hrStepCombo", m.Type);
Assert.Equal(time, m.OccurrenceTimeUtc);
Assert.Equal("abc", m.DeviceId);
Assert.Null(m.PatientId);
Assert.Null(m.EncounterId);
Assert.Null(m.CorrelationId);
Assert.Collection(
m.Properties,
p =>
{
Assert.Equal("hr", p.Name);
Assert.Equal("60", p.Value);
},
p =>
{
Assert.Equal("steps", p.Name);
Assert.Equal("2", p.Value);
});
});
}
public void GivenSingleValueTemplateAndNullTimestampToken_WhenGetMeasurements_Then_ExceptionIsThrown(IContentTemplate contentTemplate)
{
var time = DateTime.UtcNow;
var token = JToken.FromObject(new { heartrate = "60", device = "abc", date = (DateTime?)null });
Assert.Throws <IncompatibleDataException>(() => contentTemplate.GetMeasurements(token).ToArray());
}
public void GivenTemplateAndSingleMultiValueValidToken_WhenGetMeasurements_ThenSingleMeasurementReturned_Test(string eventJson)
{
var evt = EventDataTestHelper.BuildEventFromJson(eventJson);
var token = new EventDataWithJsonBodyToJTokenConverter().Convert(evt);
var result = MultiValueTemplate.GetMeasurements(token).ToArray();
Assert.NotNull(result);
Assert.Collection(result, m =>
{
Assert.Equal("bloodpressure", m.Type);
Assert.Equal(evt.Properties["iothub-creation-time-utc"], m.OccurrenceTimeUtc);
Assert.Equal(evt.SystemProperties["iothub-connection-device-id"], m.DeviceId);
Assert.Collection(
m.Properties,
p =>
{
Assert.Equal("systolic", p.Name);
Assert.Equal("111", p.Value);
},
p =>
{
Assert.Equal("diastolic", p.Name);
Assert.Equal("75", p.Value);
});
});
}
public void GivenTemplateWithCorrelationIdAndIdMissing_WhenGetMeasurements_ThenArgumentNullExceptionThrown_Test(IContentTemplate template)
{
var time = DateTime.UtcNow;
var token = JToken.FromObject(new { heartrate = "60", device = "abc", date = time });
var ex = Assert.Throws <IncompatibleDataException>(() => template.GetMeasurements(token).ToArray());
Assert.Contains("Unable to extract required value for [CorrelationIdExpression]", ex.Message);
}
private async Task StartConsumer(ISourceBlock <EventData> producer, IAsyncCollector <IMeasurement> collector, Func <Exception, EventData, Task <bool> > errorConsumer)
{
// Collect non operation canceled exceptions as they occur to ensure the entire data stream is processed
var exceptions = new ConcurrentBag <Exception>();
var cts = new CancellationTokenSource();
var consumer = new ActionBlock <EventData>(
async evt =>
{
try
{
_log.LogMetric(Metrics.DeviceEventProcessingLatency, (DateTime.UtcNow - evt.SystemProperties.EnqueuedTimeUtc).TotalSeconds);
var token = _converter.Convert(evt);
foreach (var measurement in _contentTemplate.GetMeasurements(token))
{
measurement.IngestionTimeUtc = evt.SystemProperties.EnqueuedTimeUtc;
await collector.AddAsync(measurement).ConfigureAwait(false);
_log.LogMetric(Metrics.NormalizedEvent, 1);
}
}
catch (OperationCanceledException)
{
cts.Cancel();
throw;
}
#pragma warning disable CA1031
catch (Exception ex)
{
if (await errorConsumer(ex, evt).ConfigureAwait(false))
{
exceptions.Add(ex);
}
}
#pragma warning restore CA1031
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = _maxParallelism, SingleProducerConstrained = true, CancellationToken = cts.Token
});
_ = producer.LinkTo(consumer, new DataflowLinkOptions {
PropagateCompletion = true
});
await consumer.Completion
.ContinueWith(
task =>
{
if (!exceptions.IsEmpty)
{
throw new AggregateException(exceptions);
}
},
cts.Token,
AsyncContinueOnSuccess,
TaskScheduler.Current)
.ConfigureAwait(false);
}
public void GivenTemplateAndInvalidToken_WhenGetMeasurements_ThenEmptyIEnumerableReturned_Test(IContentTemplate contentTemplate)
{
var time = DateTime.UtcNow;
var token = JToken.FromObject(new { foo = "60", device = "abc", date = time });
var result = contentTemplate.GetMeasurements(token).ToArray();
Assert.NotNull(result);
Assert.Empty(result);
}
public void GivenSingleValueCompoundAndTemplateAndPartialToken_WhenGetMeasurements_ThenEmptyIEnumerableReturned_Test(IContentTemplate template)
{
var time = DateTime.UtcNow;
var token = JToken.FromObject(new { heartrate = "60", device = "abc", mdate = time });
var result = template.GetMeasurements(token).ToArray();
Assert.NotNull(result);
Assert.Empty(result);
}
public void GivenMultiValueRequiredTemplateAndValidTokenWithMissingValue_WhenGetMeasurements_ThenInvalidOperationException_Test(IContentTemplate template)
{
var time = DateTime.UtcNow;
var token = JToken.FromObject(
new
{
Body = new[]
{
new { systolic = "120", device = "abc", date = time },
},
});
Assert.Throws <IncompatibleDataException>(() => template.GetMeasurements(token).ToArray());
}
private async Task StartConsumer(ISourceBlock <EventData> producer, IEnumerableAsyncCollector <IMeasurement> collector, Func <Exception, EventData, Task <bool> > errorConsumer)
{
// Collect non operation canceled exceptions as they occur to ensure the entire data stream is processed
var exceptions = new ConcurrentBag <Exception>();
var cts = new CancellationTokenSource();
var transformingConsumer = new TransformManyBlock <EventData, (string, IMeasurement)>(
async evt =>
{
var createdMeasurements = new List <(string, IMeasurement)>();
try
{
string partitionId = evt.SystemProperties.PartitionKey;
var deviceEventProcessingLatency = DateTime.UtcNow - evt.SystemProperties.EnqueuedTimeUtc;
_log.LogMetric(
IomtMetrics.DeviceEventProcessingLatency(partitionId),
deviceEventProcessingLatency.TotalSeconds);
_log.LogMetric(
IomtMetrics.DeviceEventProcessingLatencyMs(partitionId),
deviceEventProcessingLatency.TotalMilliseconds);
var token = _converter.Convert(evt);
try
{
foreach (var measurement in _contentTemplate.GetMeasurements(token))
{
measurement.IngestionTimeUtc = evt.SystemProperties.EnqueuedTimeUtc;
createdMeasurements.Add((partitionId, measurement));
}
}
catch (Exception ex)
{
// Translate all Normalization Mapping exceptions into a common type for easy identification.
throw new NormalizationDataMappingException(ex);
}
}
catch (Exception ex)
{
if (await errorConsumer(ex, evt).ConfigureAwait(false))
{
exceptions.Add(ex);
}
}
return(createdMeasurements);
});
var asyncCollectorConsumer = new ActionBlock <(string, IMeasurement)[]>(
protected virtual void ProcessDeviceEvent(JToken deviceEvent, IContentTemplate contentTemplate, DeviceResult validationResult)
{
try
{
foreach (var m in contentTemplate.GetMeasurements(deviceEvent))
{
validationResult.Measurements.Add(m);
}
if (validationResult.Measurements.Count == 0)
{
validationResult.CaptureWarning("No measurements were produced for the given device data.", ValidationCategory.NORMALIZATION, LineInfo.Default);
}
}
catch (Exception e)
{
validationResult.CaptureException(e, ValidationCategory.NORMALIZATION);
}
}
public void GivenEnrichmentTemplate_WhenGetMeasurements_ThenAllMeasurementReturned_Test(string eventJson)
{
var token = JsonConvert.DeserializeObject <JToken>(eventJson);
var result = EnrichmentTemplate.GetMeasurements(token).ToArray();
Assert.NotNull(result);
Assert.Collection(result, m =>
{
Assert.Equal("elevation", m.Type);
Assert.Equal(token["enqueuedTime"], m.OccurrenceTimeUtc);
Assert.Equal(token["deviceId"], m.DeviceId);
Assert.Collection(
m.Properties,
p =>
{
Assert.Equal("elevation", p.Name);
Assert.Equal("200m", p.Value);
});
});
}
public void GivenSingleValueOptionalContentTemplateAndValidToken_WhenGetMeasurements_ThenSingleMeasurementReturned_Test(IContentTemplate contentTemplate)
{
var time = DateTime.UtcNow;
var token = JToken.FromObject(new { heartrate = "60", device = "abc", date = time, pid = "123", eid = "789" });
var result = contentTemplate.GetMeasurements(token).ToArray();
Assert.NotNull(result);
Assert.Collection(result, m =>
{
Assert.Equal("heartrate", m.Type);
Assert.Equal(time, m.OccurrenceTimeUtc);
Assert.Equal("abc", m.DeviceId);
Assert.Equal("123", m.PatientId);
Assert.Equal("789", m.EncounterId);
Assert.Null(m.CorrelationId);
Assert.Collection(m.Properties, p =>
{
Assert.Equal("hr", p.Name);
Assert.Equal("60", p.Value);
});
});
}
public void GivenMultiValueRequiredTemplateAndValidTokenWithAllValues_WhenGetMeasurements_ThenSingleMeasurementReturned_Test(IContentTemplate template)
{
var time = DateTime.UtcNow;
var token = JToken.FromObject(
new
{
Body = new[]
{
new { systolic = "120", diastolic = "80", device = "abc", date = time },
},
});
var result = template.GetMeasurements(token).ToArray();
Assert.NotNull(result);
Assert.Collection(result, m =>
{
Assert.Equal("bloodpressure", m.Type);
Assert.Equal(time, m.OccurrenceTimeUtc);
Assert.Equal("abc", m.DeviceId);
Assert.Null(m.PatientId);
Assert.Null(m.EncounterId);
Assert.Null(m.CorrelationId);
Assert.Collection(
m.Properties,
p =>
{
Assert.Equal("systolic", p.Name);
Assert.Equal("120", p.Value);
},
p =>
{
Assert.Equal("diastolic", p.Name);
Assert.Equal("80", p.Value);
});
});
}
public void GivenTemplateWithCorrelationIdAndIdPresent_WhenGetMeasurements_ThenCorrelationIdReturn_Test(IContentTemplate template)
{
var time = DateTime.UtcNow;
var session = Guid.NewGuid().ToString();
var token = JToken.FromObject(new { heartrate = "60", device = "abc", date = time, session });
var result = template.GetMeasurements(token).ToArray();
Assert.NotNull(result);
Assert.Collection(result, m =>
{
Assert.Equal("heartrate", m.Type);
Assert.Equal(time, m.OccurrenceTimeUtc);
Assert.Equal("abc", m.DeviceId);
Assert.Null(m.PatientId);
Assert.Null(m.EncounterId);
Assert.Equal(session, m.CorrelationId);
Assert.Collection(m.Properties, p =>
{
Assert.Equal("hr", p.Name);
Assert.Equal("60", p.Value);
});
});
}