public async Task ProcessUploadsPoisonQueueHandler(
[QueueTrigger(ImportsPendingPoisonQueue)]
ImportMessage message)
{
try
{
await _dataImportService.FailImport(message.Id,
"File failed to import for unknown reason in upload processing stage.");
}
catch (Exception ex)
{
_logger.LogError(ex, $"Exception caught whilst processing ProcessUploadsPoisonQueueHandler function: {ex.StackTrace}");
}
}
public async Task CheckComplete(ImportObservationsMessage message, StatisticsDbContext context)
{
var import = await _dataImportService.GetImport(message.Id);
if (import.Status.IsFinished())
{
_logger.LogInformation($"Import for {import.File.Filename} is already finished in " +
$"state {import.Status} - not attempting to mark as completed or failed");
return;
}
if (import.Status.IsAborting())
{
_logger.LogInformation($"Import for {import.File.Filename} is trying to abort in " +
$"state {import.Status} - not attempting to mark as completed or failed, but " +
$"instead marking as {import.Status.GetFinishingStateOfAbortProcess()}, the final " +
$"state of the aborting process");
await _dataImportService.UpdateStatus(message.Id, import.Status.GetFinishingStateOfAbortProcess(), 100);
return;
}
if (import.NumBatches == 1 || await _batchService.GetNumBatchesRemaining(import.File) == 0)
{
var observationCount = context.Observation.Count(o => o.SubjectId.Equals(import.SubjectId));
if (!observationCount.Equals(import.TotalRows))
{
await _dataImportService.FailImport(message.Id,
$"Number of observations inserted ({observationCount}) " +
$"does not equal that expected ({import.TotalRows}) : Please delete & retry");
}
else
{
if (import.Errors.Count == 0)
{
await _dataImportService.UpdateStatus(message.Id, COMPLETE, 100);
}
else
{
await _dataImportService.UpdateStatus(message.Id, FAILED, 100);
}
}
}
else
{
var numBatchesRemaining = await _batchService.GetNumBatchesRemaining(import.File);
var percentageComplete = (double)(import.NumBatches - numBatchesRemaining) / import.NumBatches * 100;
await _dataImportService.UpdateStatus(message.Id, STAGE_4, percentageComplete);
}
}
public async Task ProcessStage1(Guid importId, ExecutionContext executionContext)
{
await _validatorService.Validate(importId, executionContext)
.OnSuccessDo(async result =>
{
await _dataImportService.Update(importId,
rowsPerBatch: result.RowsPerBatch,
totalRows: result.ImportableRowCount,
numBatches: result.NumBatches,
geographicLevels: result.GeographicLevels);
})
.OnFailureDo(async errors =>
{
await _dataImportService.FailImport(importId, errors);
_logger.LogError($"Import {importId} FAILED ...check log");
});
}
public async Task ProcessUploads(
[QueueTrigger(ImportsPendingQueue)] ImportMessage message,
ExecutionContext executionContext,
[Queue(ImportsPendingQueue)] ICollector <ImportMessage> importStagesMessageQueue,
[Queue(ImportsAvailableQueue)] ICollector <ImportObservationsMessage> importObservationsMessageQueue
)
{
try
{
var import = await _dataImportService.GetImport(message.Id);
_logger.LogInformation($"Processor Function processing import message for " +
$"{import.File.Filename} at stage {import.Status}");
switch (import.Status)
{
case DataImportStatus.CANCELLING:
_logger.LogInformation($"Import for {import.File.Filename} is in the process of being " +
"cancelled, so not processing to the next import stage - marking as " +
"CANCELLED");
await _dataImportService.UpdateStatus(import.Id, DataImportStatus.CANCELLED, 100);
break;
case DataImportStatus.CANCELLED:
_logger.LogInformation($"Import for {import.File.Filename} is cancelled, so not " +
"processing any further");
break;
case DataImportStatus.QUEUED:
case DataImportStatus.PROCESSING_ARCHIVE_FILE:
{
if (import.ZipFile != null)
{
_logger.LogInformation($"Unpacking archive for {import.ZipFile.Filename}");
await _processorService.ProcessUnpackingArchive(import.Id);
}
await _dataImportService.UpdateStatus(import.Id, DataImportStatus.STAGE_1, 0);
importStagesMessageQueue.Add(message);
break;
}
case DataImportStatus.STAGE_1:
await _processorService.ProcessStage1(import.Id, executionContext);
await _dataImportService.UpdateStatus(import.Id, DataImportStatus.STAGE_2, 0);
importStagesMessageQueue.Add(message);
break;
case DataImportStatus.STAGE_2:
await _processorService.ProcessStage2(import.Id);
await _dataImportService.UpdateStatus(import.Id, DataImportStatus.STAGE_3, 0);
importStagesMessageQueue.Add(message);
break;
case DataImportStatus.STAGE_3:
await _processorService.ProcessStage3(import.Id);
await _dataImportService.UpdateStatus(import.Id, DataImportStatus.STAGE_4, 0);
importStagesMessageQueue.Add(message);
break;
case DataImportStatus.STAGE_4:
await _processorService.ProcessStage4Messages(import.Id, importObservationsMessageQueue);
break;
}
}
catch (Exception e)
{
var ex = GetInnerException(e);
_logger.LogError(ex, $"{GetType().Name} function FAILED for Import: " +
$"{message.Id} : {ex.Message}");
_logger.LogError(ex.StackTrace);
await _dataImportService.FailImport(message.Id);
}
}
