/// <summary>
/// There are instances of validation issues in the legacy data where we've resolved to remove the offending
/// data points.
/// As this is a data-lossy action, we want to perform it here (rather than at sql script level), to ensure that
/// it is recorded in the migration log
/// </summary>
private void CleanData(Notification notification,
PerformContext context,
string requestId)
{
var missingDateResults = notification.TestData.ManualTestResults
.Where(result => !result.TestDate.HasValue)
.ToList();
missingDateResults.ForEach(result =>
{
var missingDateMessage =
$"Notification {notification.LegacyId} had test results without a date set. The notification will be imported without this test record.";
_logger.LogWarning(context, requestId, missingDateMessage);
notification.TestData.ManualTestResults.Remove(result);
});
var dateInFutureResults = notification.TestData.ManualTestResults
.Where(result => result.TestDate > DateTime.Today)
.ToList();
dateInFutureResults.ForEach(result =>
{
var dateInFutureMessage =
$"Notification {notification.LegacyId} had test results with date set in future. The notification will be imported without this test record.";
_logger.LogWarning(context, requestId, dateInFutureMessage);
notification.TestData.ManualTestResults.Remove(result);
});
var missingResults = notification.TestData.ManualTestResults
.Where(result => result.Result == null)
.ToList();
missingResults.ForEach(result =>
{
var missingResultMessage =
$"Notification {notification.LegacyId} had test results without a result recorded. The notification will be imported without this test record.";
_logger.LogWarning(context, requestId, missingResultMessage);
notification.TestData.ManualTestResults.Remove(result);
});
// After filtering out invalid tests, we might have none left
if (!notification.TestData.ManualTestResults.Any())
{
notification.TestData.HasTestCarriedOut = false;
}
if (ValidateObject(notification.ContactTracing).Any())
{
var message =
$"Notification {notification.LegacyId} invalid contact tracing figures. The notification will be imported without contact tracing data.";
_logger.LogWarning(context, requestId, message);
notification.ContactTracing = new ContactTracing();
}
}
private async Task <ImportResult> ValidateAndImportNotificationGroupAsync(PerformContext context, string requestId, List <Notification> notifications)
{
var patientName = notifications.First().PatientDetails.FullName;
var importResult = new ImportResult(patientName);
_logger.LogInformation(context, requestId, $"{notifications.Count} notifications found to import for {patientName}");
// Verify that no repeated NotificationIds have returned
var ids = notifications.Select(n => n.LegacyId).ToList();
if (ids.Distinct().Count() != ids.Count)
{
var errorMessage = $"Duplicate records found ({String.Join(',', ids)}) - aborting import for {patientName}";
importResult.AddGroupError(errorMessage);
_logger.LogImportFailure(context, requestId, errorMessage);
return(importResult);
}
bool isAnyNotificationInvalid = false;
foreach (var notification in notifications)
{
var linkedNotificationId = notification.LegacyId;
_logger.LogInformation(context, requestId, $"Validating notification with Id={linkedNotificationId}");
var validationErrors = await _importValidator.CleanAndValidateNotification(context,
requestId,
notification);
if (!validationErrors.Any())
{
_logger.LogInformation(context, requestId, "No validation errors found");
importResult.AddValidNotification(linkedNotificationId);
}
else
{
isAnyNotificationInvalid = true;
importResult.AddValidationErrorsMessages(linkedNotificationId, validationErrors);
_logger.LogWarning(context, requestId, $"{validationErrors.Count} validation errors found for notification with Id={linkedNotificationId}:");
foreach (var validationError in validationErrors)
{
_logger.LogWarning(context, requestId, validationError.ErrorMessage);
}
}
}
if (isAnyNotificationInvalid)
{
_logger.LogImportFailure(context, requestId, $"Terminating importing notifications for {patientName} due to validation errors");
return(importResult);
}
_logger.LogSuccess(context, requestId, $"Importing {notifications.Count} valid notifications");
try
{
var savedNotifications = await _notificationImportRepository.AddLinkedNotificationsAsync(notifications);
await _migratedNotificationsMarker.MarkNotificationsAsImportedAsync(savedNotifications);
importResult.NtbsIds = savedNotifications.ToDictionary(x => x.LegacyId, x => x.NotificationId);
await ImportReferenceLabResultsAsync(context, requestId, savedNotifications, importResult);
var newIdsString = string.Join(" ,", savedNotifications.Select(x => x.NotificationId));
_logger.LogSuccess(context, requestId, $"Imported notifications have following Ids: {newIdsString}");
_logger.LogInformation(context, requestId, $"Finished importing notification for {patientName}");
}
catch (MarkingNotificationsAsImportedFailedException e)
{
Log.Error(e, e.Message);
_logger.LogWarning(context, requestId, message: e.Message);
importResult.AddGroupError($"{e.Message}: {e.StackTrace}");
}
catch (Exception e)
{
Log.Error(e, e.Message);
_logger.LogImportFailure(context, requestId, message: $"Failed to save notification for {patientName} or mark it as imported ", e);
importResult.AddGroupError($"{e.Message}: {e.StackTrace}");
}
return(importResult);
}