public TObservation MergeObservation(ILookupTemplate <IFhirTemplate> lookup, IObservationGroup observationGroup, TObservation existingObservation)
{
EnsureArg.IsNotNull(observationGroup, nameof(observationGroup));
var(template, processor) = GetTemplateAndProcessor(observationGroup.Name, lookup);
return(processor.MergeObservation(template, observationGroup, existingObservation));
}
public IEnumerable <IObservationGroup> CreateObservationGroups(ILookupTemplate <IFhirTemplate> lookup, IMeasurementGroup measurementGroup)
{
EnsureArg.IsNotNull(measurementGroup, nameof(measurementGroup));
var(template, processor) = GetTemplateAndProcessor(measurementGroup.MeasureType, lookup);
return(processor.CreateObservationGroups(template, measurementGroup));
}
private void CheckForTemplateCompatibility(IContentTemplate contentTemplate, ILookupTemplate <IFhirTemplate> fhirTemplate, TemplateResult validationResult)
{
var deviceTemplates = new List <MeasurementExtractor>();
var fhirTemplates = new List <CodeValueFhirTemplate>();
var availableFhirTemplates = string.Empty;
// TODO: Confirm that outer template factories are always collections for both Device and Fhir Mappings. This implies that
// customers must always wrap their templates inside of a CollectionXXX Template.
if (contentTemplate is CollectionContentTemplate collectionContentTemplate)
{
deviceTemplates.AddRange(collectionContentTemplate.Templates.Select(t => t as MeasurementExtractor));
}
if (fhirTemplate is FhirLookupTemplate fhirLookupTemplate)
{
fhirTemplates.AddRange(fhirLookupTemplate.Templates.Select(t => t as CodeValueFhirTemplate));
availableFhirTemplates = string.Join(" ,", fhirTemplates.Select(t => t.TypeName));
}
foreach (var extractor in deviceTemplates)
{
try
{
var innerTemplate = extractor.Template;
var matchingFhirTemplate = fhirTemplate.GetTemplate(innerTemplate.TypeName) as CodeValueFhirTemplate;
var availableFhirValueNames = new HashSet <string>(GetFhirValues(matchingFhirTemplate).Select(v => v.ValueName));
var availableFhirValueNamesDisplay = string.Join(" ,", availableFhirValueNames);
// Ensure all values are present
if (extractor.Template.Values != null)
{
foreach (var v in extractor.Template.Values)
{
if (!availableFhirValueNames.Contains(v.ValueName))
{
validationResult.CaptureWarning(
$"The value [{v.ValueName}] in Device Mapping [{extractor.Template.TypeName}] is not represented within the Fhir Template of type [{innerTemplate.TypeName}]. Available values are: [{availableFhirValueNamesDisplay}]. No value will appear inside of Observations.",
ValidationCategory.FHIRTRANSFORMATION,
LineInfo.Default);
}
}
}
}
catch (TemplateNotFoundException)
{
validationResult.CaptureWarning(
$"No matching Fhir Template exists for Device Mapping [{extractor.Template.TypeName}]. Ensure case matches. Available Fhir Templates: [{availableFhirTemplates}].",
ValidationCategory.FHIRTRANSFORMATION,
LineInfo.Default);
}
catch (Exception e)
{
validationResult.CaptureException(e, ValidationCategory.FHIRTRANSFORMATION);
}
}
}
public override async Task ProcessAsync(ILookupTemplate <IFhirTemplate> config, IMeasurementGroup data, Func <Exception, IMeasurementGroup, Task <bool> > errorConsumer = null)
{
// Get required ids
var ids = await ResourceIdentityService.ResolveResourceIdentitiesAsync(data).ConfigureAwait(false);
var grps = _fhirTemplateProcessor.CreateObservationGroups(config, data);
foreach (var grp in grps)
{
_ = await SaveObservationAsync(config, grp, ids).ConfigureAwait(false);
}
}
public virtual async Task <string> SaveObservationAsync(ILookupTemplate <IFhirTemplate> config, IObservationGroup observationGroup, IDictionary <ResourceType, string> ids)
{
var identifier = GenerateObservationIdentifier(observationGroup, ids);
var cacheKey = $"{identifier.System}|{identifier.Value}";
if (!_observationCache.TryGetValue(cacheKey, out Model.Observation existingObservation))
{
existingObservation = await GetObservationFromServerAsync(identifier).ConfigureAwait(false);
}
Model.Observation result;
if (existingObservation == null)
{
var newObservation = GenerateObservation(config, observationGroup, identifier, ids);
result = await _client.CreateAsync(newObservation).ConfigureAwait(false);
_logger.LogMetric(IomtMetrics.FhirResourceSaved(ResourceType.Observation, ResourceOperation.Created), 1);
}
else
{
var policyResult = await Policy <Model.Observation>
.Handle <FhirOperationException>(ex => ex.Status == System.Net.HttpStatusCode.Conflict || ex.Status == System.Net.HttpStatusCode.PreconditionFailed)
.RetryAsync(2, async(polyRes, attempt) =>
{
existingObservation = await GetObservationFromServerAsync(identifier).ConfigureAwait(false);
})
.ExecuteAndCaptureAsync(async() =>
{
var mergedObservation = MergeObservation(config, existingObservation, observationGroup);
return(await _client.UpdateAsync(mergedObservation, versionAware: true).ConfigureAwait(false));
}).ConfigureAwait(false);
var exception = policyResult.FinalException;
if (exception != null)
{
throw exception;
}
result = policyResult.Result;
_logger.LogMetric(IomtMetrics.FhirResourceSaved(ResourceType.Observation, ResourceOperation.Updated), 1);
}
_observationCache.CreateEntry(cacheKey)
.SetAbsoluteExpiration(DateTimeOffset.UtcNow.AddHours(1))
.SetSize(1)
.SetValue(result)
.Dispose();
return(result.Id);
}
public virtual async Task <string> SaveObservationAsync(ILookupTemplate <IFhirTemplate> config, IObservationGroup observationGroup, IDictionary <ResourceType, string> ids)
{
var identifier = GenerateObservationIdentifier(observationGroup, ids);
var cacheKey = $"{identifier.System}|{identifier.Value}";
if (!_observationCache.TryGetValue(cacheKey, out Model.Observation existingObservation))
{
existingObservation = await GetObservationFromServerAsync(identifier).ConfigureAwait(false);
}
Model.Observation result;
if (existingObservation == null)
{
var newObservation = GenerateObservation(config, observationGroup, identifier, ids);
result = await _client.CreateAsync <Model.Observation>(newObservation).ConfigureAwait(false);
}
else
{
var policyResult = await Policy <Model.Observation>
.Handle <FhirOperationException>(ex => ex.Status == System.Net.HttpStatusCode.Conflict)
.FallbackAsync(async ct =>
{
var refreshObservation = await GetObservationFromServerAsync(identifier).ConfigureAwait(false);
var mergedObservation = MergeObservation(config, refreshObservation, observationGroup);
return(await _client.UpdateAsync(mergedObservation, versionAware: true).ConfigureAwait(false));
})
.ExecuteAndCaptureAsync(async() =>
{
var mergedObservation = MergeObservation(config, existingObservation, observationGroup);
return(await _client.UpdateAsync(mergedObservation, versionAware: true).ConfigureAwait(false));
}).ConfigureAwait(false);
result = policyResult.Result;
}
_observationCache.CreateEntry(cacheKey)
.SetAbsoluteExpiration(DateTimeOffset.UtcNow.AddHours(1))
.SetSize(1)
.SetValue(result)
.Dispose();
return(result.Id);
}
public ValidationResult PerformValidation(
IEnumerable <JToken> deviceEvents,
string deviceMappingContent,
string fhirMappingContent,
bool aggregateDeviceEvents = false)
{
if (string.IsNullOrWhiteSpace(deviceMappingContent) && string.IsNullOrWhiteSpace(fhirMappingContent))
{
throw new ArgumentException($"At least one of [{nameof(deviceMappingContent)}] or [{nameof(fhirMappingContent)}] must be provided");
}
var validationResult = new ValidationResult();
IContentTemplate contentTemplate = null;
ILookupTemplate <IFhirTemplate> fhirTemplate = null;
if (!string.IsNullOrEmpty(deviceMappingContent))
{
contentTemplate = LoadDeviceTemplate(deviceMappingContent, validationResult.TemplateResult);
}
if (!string.IsNullOrEmpty(fhirMappingContent))
{
fhirTemplate = LoadFhirTemplate(fhirMappingContent, validationResult.TemplateResult);
}
if (contentTemplate != null && fhirTemplate != null)
{
CheckForTemplateCompatibility(contentTemplate, fhirTemplate, validationResult.TemplateResult);
}
if (validationResult.TemplateResult.GetErrors(ErrorLevel.ERROR).Count() > 0)
{
// Fail early since there are errors with the template.
return(validationResult);
}
ValidateDeviceEvents(deviceEvents, contentTemplate, fhirTemplate, validationResult, aggregateDeviceEvents);
return(validationResult);
}
public virtual Model.Observation GenerateObservation(ILookupTemplate <IFhirTemplate> config, IObservationGroup grp, Model.Identifier observationId, IDictionary <ResourceType, string> ids)
{
EnsureArg.IsNotNull(grp, nameof(grp));
EnsureArg.IsNotNull(observationId, nameof(observationId));
var patientId = Ensure.String.IsNotNullOrWhiteSpace(ids[ResourceType.Patient], nameof(ResourceType.Patient));
var deviceId = Ensure.String.IsNotNullOrWhiteSpace(ids[ResourceType.Device], nameof(ResourceType.Device));
var observation = _fhirTemplateProcessor.CreateObservation(config, grp);
observation.Subject = patientId.ToReference <Model.Patient>();
observation.Device = deviceId.ToReference <Model.Device>();
observation.Identifier = new List <Model.Identifier> {
observationId
};
if (ids.TryGetValue(ResourceType.Encounter, out string encounterId))
{
observation.Encounter = encounterId.ToReference <Model.Encounter>();
}
return(observation);
}
protected virtual void ProcessNormalizedEvent(Measurement normalizedEvent, ILookupTemplate <IFhirTemplate> fhirTemplate, DeviceResult validationResult)
{
var measurementGroup = new MeasurementGroup
{
MeasureType = normalizedEvent.Type,
CorrelationId = normalizedEvent.CorrelationId,
DeviceId = normalizedEvent.DeviceId,
EncounterId = normalizedEvent.EncounterId,
PatientId = normalizedEvent.PatientId,
Data = new List <Measurement>()
{
normalizedEvent
},
};
try
{
// Convert Measurement to Observation Group
var observationGroup = _fhirTemplateProcessor.CreateObservationGroups(fhirTemplate, measurementGroup).First();
// Build HL7 Observation
validationResult.Observations.Add(_fhirTemplateProcessor.CreateObservation(fhirTemplate, observationGroup));
}
catch (TemplateNotFoundException e)
{
validationResult.CaptureError(
$"No Fhir Template exists with the type name [{e.Message}]. Ensure that all Fhir Template type names match Device Mapping type names (including casing)",
ErrorLevel.ERROR,
ValidationCategory.FHIRTRANSFORMATION,
LineInfo.Default);
}
catch (Exception e)
{
validationResult.CaptureException(e, ValidationCategory.FHIRTRANSFORMATION);
}
}
public virtual Model.Observation MergeObservation(ILookupTemplate <IFhirTemplate> config, Model.Observation observation, IObservationGroup grp)
{
return(_fhirTemplateProcessor.MergeObservation(config, grp, observation));
}
private (IFhirTemplate template, IFhirTemplateProcessor <IFhirTemplate, TObservation> processor) GetTemplateAndProcessor(string lookupValue, ILookupTemplate <IFhirTemplate> lookup)
{
EnsureArg.IsNotNullOrWhiteSpace(lookupValue, nameof(lookupValue));
EnsureArg.IsNotNull(lookup, nameof(lookup));
var template = lookup.GetTemplate(lookupValue);
var templateType = template.GetType();
if (!_registeredTemplateProcessors.TryGetValue(templateType, out var processor))
{
throw new NotSupportedException($"No processor registered for template type {templateType}.");
}
return(template, processor);
}
public override Task ProcessAsync(ILookupTemplate <IFhirTemplate> config, IMeasurementGroup data, Func <Exception, IMeasurementGroup, Task <bool> > errorConsumer = null)
{
throw new NotImplementedException();
}
private async Task ProcessMeasurementGroups(IEnumerable <IMeasurementGroup> measurementGroups, ILookupTemplate <IFhirTemplate> template, ITelemetryLogger log)
{
// Group work by device to avoid race conditions when resource creation is enabled.
var workItems = measurementGroups.GroupBy(grp => grp.DeviceId)
.Select(grp => new Func <Task>(
async() =>
{
foreach (var m in grp)
{
try
{
await _fhirImportService.ProcessAsync(template, m).ConfigureAwait(false);
}
catch (Exception ex)
{
if (!Options.ExceptionService.HandleException(ex, log))
{
throw;
}
}
}
}));
await StartWorker(workItems).ConfigureAwait(false);
}
public abstract Task ProcessAsync(ILookupTemplate <IFhirTemplate> config, IMeasurementGroup data, Func <Exception, IMeasurementGroup, Task <bool> > errorConsumer = null);
/// <summary>
/// Validates device events. This method then enriches the passed in ValidationResult object with DeviceResults.
/// </summary>
/// <param name="deviceEvents">The device events to validate</param>
/// <param name="contentTemplate">The device mapping template</param>
/// <param name="fhirTemplate">The fhir mapping template</param>
/// <param name="validationResult">The ValidationResult</param>
/// <param name="aggregateDeviceEvents">Indicates if DeviceResults should be aggregated</param>
protected virtual void ValidateDeviceEvents(
IEnumerable <JToken> deviceEvents,
IContentTemplate contentTemplate,
ILookupTemplate <IFhirTemplate> fhirTemplate,
ValidationResult validationResult,
bool aggregateDeviceEvents)
{
var aggregatedDeviceResults = new Dictionary <string, DeviceResult>();
foreach (var payload in deviceEvents)
{
if (payload != null && contentTemplate != null)
{
var deviceResult = new DeviceResult();
deviceResult.DeviceEvent = payload;
ProcessDeviceEvent(payload, contentTemplate, deviceResult);
if (fhirTemplate != null)
{
foreach (var m in deviceResult.Measurements)
{
ProcessNormalizedEvent(m, fhirTemplate, deviceResult);
}
}
if (aggregateDeviceEvents)
{
/*
* During aggregation we group DeviceEvents by the exceptions that they produce.
* This allows us to return a DeviceResult with a sample Device Event payload,
* the running count grouped DeviceEvents and the exception that they are grouped by.
*/
foreach (var exception in deviceResult.Exceptions)
{
if (aggregatedDeviceResults.TryGetValue(exception.Message, out DeviceResult result))
{
// If we've already seen this error message before, simply increment the running total
result.AggregatedCount++;
}
else
{
// Create a new DeviceResult to hold details about this new exception.
var aggregatedDeviceResult = new DeviceResult()
{
DeviceEvent = deviceResult.DeviceEvent, // A sample device event which exhibits the error
AggregatedCount = 1,
Exceptions = new List <ValidationError>()
{
exception
},
};
aggregatedDeviceResults[exception.Message] = aggregatedDeviceResult;
validationResult.DeviceResults.Add(aggregatedDeviceResult);
}
}
}
else
{
validationResult.DeviceResults.Add(deviceResult);
}
}
}
}
public virtual async Task <string> SaveObservationAsync(ILookupTemplate <IFhirTemplate> config, IObservationGroup observationGroup, IDictionary <ResourceType, string> ids)
{
var identifier = GenerateObservationIdentifier(observationGroup, ids);
var cacheKey = $"{identifier.System}|{identifier.Value}";
if (!_observationCache.TryGetValue(cacheKey, out Model.Observation existingObservation))
{
existingObservation = await GetObservationFromServerAsync(identifier).ConfigureAwait(false);
// Discovered an issue where FHIR Service is only matching on first 128 characters of the identifier. This is a temporary measure to prevent merging of different observations until a fix is available.
if (existingObservation != null && !existingObservation.Identifier.Exists(i => i.IsExactly(identifier)))
{
throw new NotSupportedException("FHIR Service returned matching observation but expected identifier was not present.");
}
}
var policyResult = await Policy <(Model.Observation observation, ResourceOperation operationType)>
.Handle <FhirException>(ex => ex.StatusCode == System.Net.HttpStatusCode.Conflict || ex.StatusCode == System.Net.HttpStatusCode.PreconditionFailed)
.RetryAsync(2, async(polyRes, attempt) =>
{
// 409 Conflict or 412 Precondition Failed can occur if the Observation.meta.versionId does not match the update request.
// This can happen if 2 independent processes are updating the same Observation simultaneously.
// or
// The update operation failed because the Observation no longer exists.
// This can happen if a cached Observation was deleted from the FHIR Server.
_logger.LogTrace("A conflict or precondition caused an Observation update to fail. Getting the most recent Observation.");
// Attempt to get the most recent version of the Observation.
existingObservation = await GetObservationFromServerAsync(identifier).ConfigureAwait(false);
// If the Observation no longer exists on the FHIR Server, it was most likely deleted.
if (existingObservation == null)
{
_logger.LogTrace("A cached version of an Observation was deleted. Creating a new Observation.");
// Remove the Observation from the cache (this version no longer exists on the FHIR Server.
_observationCache.Remove(cacheKey);
}
})
.ExecuteAndCaptureAsync(async() =>
{
if (existingObservation == null)
{
var newObservation = GenerateObservation(config, observationGroup, identifier, ids);
return(await _fhirService.CreateResourceAsync(newObservation).ConfigureAwait(false), ResourceOperation.Created);
}
// Merge the new data with the existing Observation.
var mergedObservation = MergeObservation(config, existingObservation, observationGroup);
// Check to see if there are any changes after merging and update the Status to amended if changed.
if (!existingObservation.AmendIfChanged(mergedObservation))
{
// There are no changes to the Observation - Do not update.
return(existingObservation, ResourceOperation.NoOperation);
}
// Update the Observation. Some failures will be handled in the RetryAsync block above.
return(await _fhirService.UpdateResourceAsync(mergedObservation).ConfigureAwait(false), ResourceOperation.Updated);
}).ConfigureAwait(false);
var exception = policyResult.FinalException;
if (exception != null)
{
throw exception;
}
var observation = policyResult.Result.observation;
_logger.LogMetric(IomtMetrics.FhirResourceSaved(ResourceType.Observation, policyResult.Result.operationType), 1);
_observationCache.CreateEntry(cacheKey)
.SetAbsoluteExpiration(DateTimeOffset.UtcNow.AddHours(1))
.SetSize(1)
.SetValue(observation)
.Dispose();
return(observation.Id);
}