public async void ProcessUploadsPoisonQueueHandler(
[QueueTrigger("imports-pending-poison")]
ImportMessage message,
ILogger logger
)
{
await _batchService.FailImport(
message.Release.Id, message.DataFileName,
new List <ValidationError>
{
new ValidationError("File failed to import for unknown reason in upload processing stage.")
}.AsEnumerable()
);
}
public async Task ProcessStage1(ImportMessage message, ExecutionContext executionContext)
{
var subjectData = await GetSubjectDataFromMainDataFile(message);
await _validatorService.Validate(subjectData, executionContext, message)
.OnSuccessDo(async result =>
{
message.RowsPerBatch = result.RowsPerBatch;
message.TotalRows = result.FilteredObservationCount;
message.NumBatches = result.NumBatches;
await _batchService.UpdateStoredMessage(message);
})
.OnFailureDo(async errors =>
{
await _batchService.FailImport(message.Release.Id,
message.DataFileName,
errors);
_logger.LogError($"Import FAILED for {message.DataFileName}...check log");
});
}
public async void ProcessUploads(
[QueueTrigger("imports-pending")] ImportMessage message,
ExecutionContext executionContext,
[Queue("imports-pending")] ICollector <ImportMessage> importStagesMessageQueue,
[Queue("imports-available")] ICollector <ImportObservationsMessage> importObservationsMessageQueue
)
{
try
{
var status = await _importStatusService.GetImportStatus(message.Release.Id, message.DataFileName);
_logger.LogInformation($"Processor Function processing import message for " +
$"{message.DataFileName} at stage {status.Status}");
switch (status.Status)
{
case IStatus.CANCELLING:
_logger.LogInformation($"Import for {message.DataFileName} is in the process of being " +
$"cancelled, so not processing to the next import stage - marking as " +
$"CANCELLED");
await _importStatusService.UpdateStatus(message.Release.Id, message.DataFileName, IStatus.CANCELLED,
100);
break;
case IStatus.CANCELLED:
_logger.LogInformation($"Import for {message.DataFileName} is cancelled, so not " +
$"processing any further");
break;
case IStatus.QUEUED:
case IStatus.PROCESSING_ARCHIVE_FILE:
{
if (message.ArchiveFileName != "")
{
_logger.LogInformation($"Unpacking archive for {message.DataFileName}");
await _processorService.ProcessUnpackingArchive(message);
}
await _importStatusService.UpdateStatus(message.Release.Id, message.DataFileName,
IStatus.STAGE_1);
importStagesMessageQueue.Add(message);
break;
}
case IStatus.STAGE_1:
await _processorService.ProcessStage1(message, executionContext);
await _importStatusService.UpdateStatus(message.Release.Id, message.DataFileName,
IStatus.STAGE_2);
importStagesMessageQueue.Add(message);
break;
case IStatus.STAGE_2:
await _processorService.ProcessStage2(message);
await _importStatusService.UpdateStatus(message.Release.Id, message.DataFileName,
IStatus.STAGE_3);
importStagesMessageQueue.Add(message);
break;
case IStatus.STAGE_3:
await _processorService.ProcessStage3(message);
await _importStatusService.UpdateStatus(message.Release.Id, message.DataFileName,
IStatus.STAGE_4);
importStagesMessageQueue.Add(message);
break;
case IStatus.STAGE_4:
await _processorService.ProcessStage4Messages(message, importObservationsMessageQueue);
break;
}
}
catch (Exception e)
{
var ex = GetInnerException(e);
await _batchService.FailImport(message.Release.Id,
message.DataFileName,
new List <ValidationError>
{
new ValidationError(ex.Message)
});
_logger.LogError(ex, $"{GetType().Name} function FAILED for : Datafile: " +
$"{message.DataFileName} : {ex.Message}");
_logger.LogError(ex.StackTrace);
}
}
public async Task CheckComplete(Guid releaseId, ImportObservationsMessage message, StatisticsDbContext context)
{
var import = await _importStatusService.GetImportStatus(releaseId, message.DataFileName);
if (import.IsFinished())
{
_logger.LogInformation($"Import for {message.DataFileName} is already finished in " +
$"state {import.Status} - not attempting to mark as completed or failed");
return;
}
if (import.IsAborting())
{
_logger.LogInformation($"Import for {message.DataFileName} is trying to abort in " +
$"state {import.Status} - not attempting to mark as completed or failed, but " +
$"instead marking as {import.GetFinishingStateOfAbortProcess()}, the final " +
$"state of the aborting process");
await _importStatusService.UpdateStatus(releaseId,
message.DataFileName,
import.GetFinishingStateOfAbortProcess(),
100);
return;
}
if (message.NumBatches == 1 ||
await _fileStorageService.GetNumBatchesRemaining(releaseId, message.DataFileName) == 0)
{
var observationCount = context.Observation.Count(o => o.SubjectId.Equals(message.SubjectId));
if (!observationCount.Equals(message.TotalRows))
{
await _batchService.FailImport(releaseId, message.DataFileName,
new List <ValidationError>
{
new ValidationError(
$"Number of observations inserted ({observationCount}) " +
$"does not equal that expected ({message.TotalRows}) : Please delete & retry"
)
}.AsEnumerable());
}
else
{
if (import.Errors.IsNullOrEmpty())
{
await _importStatusService.UpdateStatus(releaseId, message.DataFileName, COMPLETE, 100);
}
else
{
await _importStatusService.UpdateStatus(releaseId, message.DataFileName, FAILED, 100);
}
}
}
else
{
var numBatchesRemaining =
await _fileStorageService.GetNumBatchesRemaining(releaseId, message.DataFileName);
var percentageComplete = (double)(message.NumBatches - numBatchesRemaining) / message.NumBatches * 100;
await _importStatusService.UpdateStatus(releaseId,
message.DataFileName,
STAGE_4,
percentageComplete);
}
}
