public async Task ProcessUploadsPoisonQueueHandler( [QueueTrigger(ImportsPendingPoisonQueue)] ImportMessage message) { try { await _dataImportService.FailImport(message.Id, "File failed to import for unknown reason in upload processing stage."); } catch (Exception ex) { _logger.LogError(ex, $"Exception caught whilst processing ProcessUploadsPoisonQueueHandler function: {ex.StackTrace}"); } }
public async Task CheckComplete(ImportObservationsMessage message, StatisticsDbContext context) { var import = await _dataImportService.GetImport(message.Id); if (import.Status.IsFinished()) { _logger.LogInformation($"Import for {import.File.Filename} is already finished in " + $"state {import.Status} - not attempting to mark as completed or failed"); return; } if (import.Status.IsAborting()) { _logger.LogInformation($"Import for {import.File.Filename} is trying to abort in " + $"state {import.Status} - not attempting to mark as completed or failed, but " + $"instead marking as {import.Status.GetFinishingStateOfAbortProcess()}, the final " + $"state of the aborting process"); await _dataImportService.UpdateStatus(message.Id, import.Status.GetFinishingStateOfAbortProcess(), 100); return; } if (import.NumBatches == 1 || await _batchService.GetNumBatchesRemaining(import.File) == 0) { var observationCount = context.Observation.Count(o => o.SubjectId.Equals(import.SubjectId)); if (!observationCount.Equals(import.TotalRows)) { await _dataImportService.FailImport(message.Id, $"Number of observations inserted ({observationCount}) " + $"does not equal that expected ({import.TotalRows}) : Please delete & retry"); } else { if (import.Errors.Count == 0) { await _dataImportService.UpdateStatus(message.Id, COMPLETE, 100); } else { await _dataImportService.UpdateStatus(message.Id, FAILED, 100); } } } else { var numBatchesRemaining = await _batchService.GetNumBatchesRemaining(import.File); var percentageComplete = (double)(import.NumBatches - numBatchesRemaining) / import.NumBatches * 100; await _dataImportService.UpdateStatus(message.Id, STAGE_4, percentageComplete); } }
public async Task ProcessStage1(Guid importId, ExecutionContext executionContext) { await _validatorService.Validate(importId, executionContext) .OnSuccessDo(async result => { await _dataImportService.Update(importId, rowsPerBatch: result.RowsPerBatch, totalRows: result.ImportableRowCount, numBatches: result.NumBatches, geographicLevels: result.GeographicLevels); }) .OnFailureDo(async errors => { await _dataImportService.FailImport(importId, errors); _logger.LogError($"Import {importId} FAILED ...check log"); }); }
public async Task ProcessUploads( [QueueTrigger(ImportsPendingQueue)] ImportMessage message, ExecutionContext executionContext, [Queue(ImportsPendingQueue)] ICollector <ImportMessage> importStagesMessageQueue, [Queue(ImportsAvailableQueue)] ICollector <ImportObservationsMessage> importObservationsMessageQueue ) { try { var import = await _dataImportService.GetImport(message.Id); _logger.LogInformation($"Processor Function processing import message for " + $"{import.File.Filename} at stage {import.Status}"); switch (import.Status) { case DataImportStatus.CANCELLING: _logger.LogInformation($"Import for {import.File.Filename} is in the process of being " + "cancelled, so not processing to the next import stage - marking as " + "CANCELLED"); await _dataImportService.UpdateStatus(import.Id, DataImportStatus.CANCELLED, 100); break; case DataImportStatus.CANCELLED: _logger.LogInformation($"Import for {import.File.Filename} is cancelled, so not " + "processing any further"); break; case DataImportStatus.QUEUED: case DataImportStatus.PROCESSING_ARCHIVE_FILE: { if (import.ZipFile != null) { _logger.LogInformation($"Unpacking archive for {import.ZipFile.Filename}"); await _processorService.ProcessUnpackingArchive(import.Id); } await _dataImportService.UpdateStatus(import.Id, DataImportStatus.STAGE_1, 0); importStagesMessageQueue.Add(message); break; } case DataImportStatus.STAGE_1: await _processorService.ProcessStage1(import.Id, executionContext); await _dataImportService.UpdateStatus(import.Id, DataImportStatus.STAGE_2, 0); importStagesMessageQueue.Add(message); break; case DataImportStatus.STAGE_2: await _processorService.ProcessStage2(import.Id); await _dataImportService.UpdateStatus(import.Id, DataImportStatus.STAGE_3, 0); importStagesMessageQueue.Add(message); break; case DataImportStatus.STAGE_3: await _processorService.ProcessStage3(import.Id); await _dataImportService.UpdateStatus(import.Id, DataImportStatus.STAGE_4, 0); importStagesMessageQueue.Add(message); break; case DataImportStatus.STAGE_4: await _processorService.ProcessStage4Messages(import.Id, importObservationsMessageQueue); break; } } catch (Exception e) { var ex = GetInnerException(e); _logger.LogError(ex, $"{GetType().Name} function FAILED for Import: " + $"{message.Id} : {ex.Message}"); _logger.LogError(ex.StackTrace); await _dataImportService.FailImport(message.Id); } }