public async Task ImportObservations(ImportObservationsMessage message, StatisticsDbContext context)
{
var import = await _dataImportService.GetImport(message.Id);
_logger.LogInformation($"Importing Observations for {import.File.Filename} batchNo {message.BatchNo}");
if (import.Status.IsFinished())
{
_logger.LogInformation($"Import for {import.File.Filename} already finished with state " +
$"{import.Status} - ignoring Observations in file {message.ObservationsFilePath}");
return;
}
if (import.Status == CANCELLING)
{
_logger.LogInformation($"Import for {import.File.Filename} is " +
$"{import.Status} - ignoring Observations in file {message.ObservationsFilePath} " +
"and marking import as CANCELLED");
await _dataImportService.UpdateStatus(message.Id, CANCELLED, 100);
return;
}
var subject = await context.Subject.FindAsync(import.SubjectId);
var datafileStream = await _blobStorageService.StreamBlob(PrivateReleaseFiles, message.ObservationsFilePath);
var dataFileTable = DataTableUtils.CreateFromStream(datafileStream);
var metaFileStream = await _blobStorageService.StreamBlob(PrivateReleaseFiles, import.MetaFile.Path());
var metaFileTable = DataTableUtils.CreateFromStream(metaFileStream);
await context.Database.CreateExecutionStrategy().Execute(async() =>
{
await using var transaction = await context.Database.BeginTransactionAsync();
await _importerService.ImportObservations(
import,
dataFileTable.Columns,
dataFileTable.Rows,
subject,
_importerService.GetMeta(metaFileTable, subject, context),
message.BatchNo,
context
);
await transaction.CommitAsync();
await context.Database.CloseConnectionAsync();
});
if (import.NumBatches > 1)
{
await _blobStorageService.DeleteBlob(PrivateReleaseFiles, message.ObservationsFilePath);
}
await CheckComplete(message, context);
}
public async Task SplitDataFile(Guid importId)
{
var import = await _dataImportService.GetImport(importId);
var dataFileStream = await _blobStorageService.StreamBlob(PrivateReleaseFiles, import.File.Path());
var dataFileTable = DataTableUtils.CreateFromStream(dataFileStream);
if (dataFileTable.Rows.Count > import.RowsPerBatch)
{
_logger.LogInformation($"Splitting Datafile: {import.File.Filename}");
await SplitFiles(import, dataFileTable);
_logger.LogInformation($"Split of Datafile: {import.File.Filename} complete");
}
else
{
_logger.LogInformation($"No splitting of datafile: {import.File.Filename} was necessary");
}
}
public async Task <Either <ActionResult, DataImportViewModel> > GetDataFileImportStatus(Guid releaseId, Guid fileId)
{
return(await _persistenceHelper
.CheckEntityExists <Release>(releaseId)
.OnSuccess(_userService.CheckCanViewRelease)
.OnSuccess(async _ =>
{
// Ensure file is linked to the Release by getting the ReleaseFile first
var releaseFile = await _releaseFileRepository.Find(releaseId, fileId);
if (releaseFile == null || releaseFile.File.Type != FileType.Data)
{
return DataImportViewModel.NotFound();
}
return await _dataImportService.GetImport(fileId);
}));
}
public async Task ProcessUploads(
[QueueTrigger(ImportsPendingQueue)] ImportMessage message,
ExecutionContext executionContext,
[Queue(ImportsPendingQueue)] ICollector <ImportMessage> importStagesMessageQueue,
[Queue(ImportsAvailableQueue)] ICollector <ImportObservationsMessage> importObservationsMessageQueue
)
{
try
{
var import = await _dataImportService.GetImport(message.Id);
_logger.LogInformation($"Processor Function processing import message for " +
$"{import.File.Filename} at stage {import.Status}");
switch (import.Status)
{
case DataImportStatus.CANCELLING:
_logger.LogInformation($"Import for {import.File.Filename} is in the process of being " +
"cancelled, so not processing to the next import stage - marking as " +
"CANCELLED");
await _dataImportService.UpdateStatus(import.Id, DataImportStatus.CANCELLED, 100);
break;
case DataImportStatus.CANCELLED:
_logger.LogInformation($"Import for {import.File.Filename} is cancelled, so not " +
"processing any further");
break;
case DataImportStatus.QUEUED:
case DataImportStatus.PROCESSING_ARCHIVE_FILE:
{
if (import.ZipFile != null)
{
_logger.LogInformation($"Unpacking archive for {import.ZipFile.Filename}");
await _processorService.ProcessUnpackingArchive(import.Id);
}
await _dataImportService.UpdateStatus(import.Id, DataImportStatus.STAGE_1, 0);
importStagesMessageQueue.Add(message);
break;
}
case DataImportStatus.STAGE_1:
await _processorService.ProcessStage1(import.Id, executionContext);
await _dataImportService.UpdateStatus(import.Id, DataImportStatus.STAGE_2, 0);
importStagesMessageQueue.Add(message);
break;
case DataImportStatus.STAGE_2:
await _processorService.ProcessStage2(import.Id);
await _dataImportService.UpdateStatus(import.Id, DataImportStatus.STAGE_3, 0);
importStagesMessageQueue.Add(message);
break;
case DataImportStatus.STAGE_3:
await _processorService.ProcessStage3(import.Id);
await _dataImportService.UpdateStatus(import.Id, DataImportStatus.STAGE_4, 0);
importStagesMessageQueue.Add(message);
break;
case DataImportStatus.STAGE_4:
await _processorService.ProcessStage4Messages(import.Id, importObservationsMessageQueue);
break;
}
}
catch (Exception e)
{
var ex = GetInnerException(e);
_logger.LogError(ex, $"{GetType().Name} function FAILED for Import: " +
$"{message.Id} : {ex.Message}");
_logger.LogError(ex.StackTrace);
await _dataImportService.FailImport(message.Id);
}
}
public async Task ProcessUnpackingArchive(Guid importId)
{
var import = await _dataImportService.GetImport(importId);
await _dataArchiveService.ExtractDataFiles(import);
}