public async Task GivenImportTaskInput_WhenExceptionThrowForLoad_ThenRetriableExceptionShouldBeThrow()
{
ImportProcessingTaskInputData inputData = GetInputData();
ImportProcessingProgress progress = new ImportProcessingProgress();
IImportResourceLoader loader = Substitute.For <IImportResourceLoader>();
IResourceBulkImporter importer = Substitute.For <IResourceBulkImporter>();
IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>();
IImportErrorStoreFactory importErrorStoreFactory = Substitute.For <IImportErrorStoreFactory>();
IContextUpdater contextUpdater = Substitute.For <IContextUpdater>();
RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
ILoggerFactory loggerFactory = new NullLoggerFactory();
loader.LoadResources(Arg.Any <string>(), Arg.Any <long>(), Arg.Any <string>(), Arg.Any <Func <long, long> >(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
long startIndex = (long)callInfo[1];
Func <long, long> idGenerator = (Func <long, long>)callInfo[3];
Channel <ImportResource> resourceChannel = Channel.CreateUnbounded <ImportResource>();
Task loadTask = Task.Run(() =>
{
try
{
throw new InvalidOperationException();
}
finally
{
resourceChannel.Writer.Complete();
}
});
return(resourceChannel, loadTask);
});
ImportProcessingTask task = new ImportProcessingTask(
inputData,
progress,
loader,
importer,
importErrorStoreFactory,
contextUpdater,
contextAccessor,
loggerFactory);
await Assert.ThrowsAsync <RetriableTaskException>(() => task.ExecuteAsync());
}
public async Task GivenImportTaskInput_WhenExceptionThrowForImport_ThenContextShouldBeUpdatedBeforeFailure()
{
long currentIndex = 100;
ImportProcessingTaskInputData inputData = GetInputData();
ImportProcessingProgress progress = new ImportProcessingProgress();
IImportResourceLoader loader = Substitute.For <IImportResourceLoader>();
IResourceBulkImporter importer = Substitute.For <IResourceBulkImporter>();
IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>();
IImportErrorStoreFactory importErrorStoreFactory = Substitute.For <IImportErrorStoreFactory>();
IContextUpdater contextUpdater = Substitute.For <IContextUpdater>();
RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
ILoggerFactory loggerFactory = new NullLoggerFactory();
loader.LoadResources(Arg.Any <string>(), Arg.Any <long>(), Arg.Any <string>(), Arg.Any <Func <long, long> >(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
long startIndex = (long)callInfo[1];
Func <long, long> idGenerator = (Func <long, long>)callInfo[3];
Channel <ImportResource> resourceChannel = Channel.CreateUnbounded <ImportResource>();
resourceChannel.Writer.Complete();
return(resourceChannel, Task.CompletedTask);
});
importer.Import(Arg.Any <Channel <ImportResource> >(), Arg.Any <IImportErrorStore>(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
Channel <ImportProcessingProgress> progressChannel = Channel.CreateUnbounded <ImportProcessingProgress>();
Task loadTask = Task.Run(async() =>
{
try
{
ImportProcessingProgress progress = new ImportProcessingProgress();
progress.CurrentIndex = currentIndex;
await progressChannel.Writer.WriteAsync(progress);
throw new InvalidOperationException();
}
finally
{
progressChannel.Writer.Complete();
}
});
return(progressChannel, loadTask);
});
string context = null;
contextUpdater.UpdateContextAsync(Arg.Any <string>(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
context = (string)callInfo[0];
return(Task.CompletedTask);
});
ImportProcessingTask task = new ImportProcessingTask(
inputData,
progress,
loader,
importer,
importErrorStoreFactory,
contextUpdater,
contextAccessor,
loggerFactory);
await Assert.ThrowsAsync <RetriableTaskException>(() => task.ExecuteAsync());
ImportProcessingProgress progressForContext = JsonConvert.DeserializeObject <ImportProcessingProgress>(context);
Assert.Equal(progressForContext.CurrentIndex, currentIndex);
}
private static async Task VerifyCommonImportTaskAsync(ImportProcessingTaskInputData inputData, ImportProcessingProgress progress)
{
long startIndexFromProgress = progress.CurrentIndex;
long succeedCountFromProgress = progress.SucceedImportCount;
long failedCountFromProgress = progress.FailedImportCount;
IImportResourceLoader loader = Substitute.For <IImportResourceLoader>();
IResourceBulkImporter importer = Substitute.For <IResourceBulkImporter>();
IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>();
IImportErrorStoreFactory importErrorStoreFactory = Substitute.For <IImportErrorStoreFactory>();
IContextUpdater contextUpdater = Substitute.For <IContextUpdater>();
RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
ILoggerFactory loggerFactory = new NullLoggerFactory();
long cleanStart = -1;
long cleanEnd = -1;
importer.CleanResourceAsync(Arg.Any <ImportProcessingTaskInputData>(), Arg.Any <ImportProcessingProgress>(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
var inputData = (ImportProcessingTaskInputData)callInfo[0];
var progress = (ImportProcessingProgress)callInfo[1];
long beginSequenceId = inputData.BeginSequenceId;
long endSequenceId = inputData.EndSequenceId;
long endIndex = progress.CurrentIndex;
cleanStart = beginSequenceId + endIndex;
cleanEnd = endSequenceId;
return(Task.CompletedTask);
});
loader.LoadResources(Arg.Any <string>(), Arg.Any <long>(), Arg.Any <string>(), Arg.Any <Func <long, long> >(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
long startIndex = (long)callInfo[1];
Func <long, long> idGenerator = (Func <long, long>)callInfo[3];
Channel <ImportResource> resourceChannel = Channel.CreateUnbounded <ImportResource>();
Task loadTask = Task.Run(async() =>
{
ResourceWrapper resourceWrapper = new ResourceWrapper(
Guid.NewGuid().ToString(),
"0",
"Dummy",
new RawResource(Guid.NewGuid().ToString(), Fhir.Core.Models.FhirResourceFormat.Json, true),
new ResourceRequest("POST"),
DateTimeOffset.UtcNow,
false,
null,
null,
null,
"SearchParam");
await resourceChannel.Writer.WriteAsync(new ImportResource(idGenerator(startIndex), startIndex, resourceWrapper));
await resourceChannel.Writer.WriteAsync(new ImportResource(idGenerator(startIndex + 1), startIndex + 1, "Error"));
resourceChannel.Writer.Complete();
});
return(resourceChannel, loadTask);
});
importer.Import(Arg.Any <Channel <ImportResource> >(), Arg.Any <IImportErrorStore>(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
Channel <ImportResource> resourceChannel = (Channel <ImportResource>)callInfo[0];
Channel <ImportProcessingProgress> progressChannel = Channel.CreateUnbounded <ImportProcessingProgress>();
Task loadTask = Task.Run(async() =>
{
ImportProcessingProgress progress = new ImportProcessingProgress();
await foreach (ImportResource resource in resourceChannel.Reader.ReadAllAsync())
{
if (string.IsNullOrEmpty(resource.ImportError))
{
progress.SucceedImportCount++;
}
else
{
progress.FailedImportCount++;
}
progress.CurrentIndex = resource.Index + 1;
}
await progressChannel.Writer.WriteAsync(progress);
progressChannel.Writer.Complete();
});
return(progressChannel, loadTask);
});
string context = null;
contextUpdater.UpdateContextAsync(Arg.Any <string>(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
context = (string)callInfo[0];
return(Task.CompletedTask);
});
progress.NeedCleanData = true;
ImportProcessingTask task = new ImportProcessingTask(
inputData,
progress,
loader,
importer,
importErrorStoreFactory,
contextUpdater,
contextAccessor,
loggerFactory);
TaskResultData taskResult = await task.ExecuteAsync();
Assert.Equal(TaskResult.Success, taskResult.Result);
ImportProcessingTaskResult result = JsonConvert.DeserializeObject <ImportProcessingTaskResult>(taskResult.ResultData);
Assert.Equal(1 + failedCountFromProgress, result.FailedCount);
Assert.Equal(1 + succeedCountFromProgress, result.SucceedCount);
ImportProcessingProgress progressForContext = JsonConvert.DeserializeObject <ImportProcessingProgress>(context);
Assert.Equal(progressForContext.SucceedImportCount, result.SucceedCount);
Assert.Equal(progressForContext.FailedImportCount, result.FailedCount);
Assert.Equal(startIndexFromProgress + 2, progressForContext.CurrentIndex);
Assert.Equal(startIndexFromProgress, cleanStart);
Assert.Equal(inputData.EndSequenceId, cleanEnd);
}
public async Task <TaskResultData> ExecuteAsync()
{
var fhirRequestContext = new FhirRequestContext(
method: "Import",
uriString: _inputData.UriString,
baseUriString: _inputData.BaseUriString,
correlationId: _inputData.TaskId,
requestHeaders: new Dictionary <string, StringValues>(),
responseHeaders: new Dictionary <string, StringValues>())
{
IsBackgroundTask = true,
};
_contextAccessor.RequestContext = fhirRequestContext;
CancellationToken cancellationToken = _cancellationTokenSource.Token;
long succeedImportCount = _importProgress.SucceedImportCount;
long failedImportCount = _importProgress.FailedImportCount;
ImportProcessingTaskResult result = new ImportProcessingTaskResult();
result.ResourceType = _inputData.ResourceType;
try
{
if (cancellationToken.IsCancellationRequested)
{
throw new OperationCanceledException();
}
Func <long, long> sequenceIdGenerator = (index) => _inputData.BeginSequenceId + index;
// Clean resources before import start
await _resourceBulkImporter.CleanResourceAsync(_inputData, _importProgress, cancellationToken);
_importProgress.NeedCleanData = true;
await _contextUpdater.UpdateContextAsync(JsonConvert.SerializeObject(_importProgress), cancellationToken);
// Initialize error store
IImportErrorStore importErrorStore = await _importErrorStoreFactory.InitializeAsync(GetErrorFileName(), cancellationToken);
result.ErrorLogLocation = importErrorStore.ErrorFileLocation;
// Load and parse resource from bulk resource
(Channel <ImportResource> importResourceChannel, Task loadTask) = _importResourceLoader.LoadResources(_inputData.ResourceLocation, _importProgress.CurrentIndex, _inputData.ResourceType, sequenceIdGenerator, cancellationToken);
// Import to data store
(Channel <ImportProcessingProgress> progressChannel, Task importTask) = _resourceBulkImporter.Import(importResourceChannel, importErrorStore, cancellationToken);
// Update progress for checkpoints
await foreach (ImportProcessingProgress progress in progressChannel.Reader.ReadAllAsync())
{
if (cancellationToken.IsCancellationRequested)
{
throw new OperationCanceledException("Import task is canceled by user.");
}
_importProgress.SucceedImportCount = progress.SucceedImportCount + succeedImportCount;
_importProgress.FailedImportCount = progress.FailedImportCount + failedImportCount;
_importProgress.CurrentIndex = progress.CurrentIndex;
result.SucceedCount = _importProgress.SucceedImportCount;
result.FailedCount = _importProgress.FailedImportCount;
_logger.LogInformation("Import task progress: {0}", JsonConvert.SerializeObject(_importProgress));
try
{
await _contextUpdater.UpdateContextAsync(JsonConvert.SerializeObject(_importProgress), cancellationToken);
}
catch (Exception ex)
{
// ignore exception for progresss update
_logger.LogInformation(ex, "Failed to update context.");
}
}
// Pop up exception during load & import
// Put import task before load task for resource channel full and blocking issue.
try
{
await importTask;
}
catch (TaskCanceledException)
{
throw;
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to import data.");
throw new RetriableTaskException("Failed to import data.", ex);
}
try
{
await loadTask;
}
catch (TaskCanceledException)
{
throw;
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to load data.");
throw new RetriableTaskException("Failed to load data", ex);
}
return(new TaskResultData(TaskResult.Success, JsonConvert.SerializeObject(result)));
}
catch (TaskCanceledException canceledEx)
{
_logger.LogInformation(canceledEx, "Data processing task is canceled.");
await CleanResourceForFailureAsync(canceledEx);
return(new TaskResultData(TaskResult.Canceled, JsonConvert.SerializeObject(result)));
}
catch (OperationCanceledException canceledEx)
{
_logger.LogInformation(canceledEx, "Data processing task is canceled.");
await CleanResourceForFailureAsync(canceledEx);
return(new TaskResultData(TaskResult.Canceled, JsonConvert.SerializeObject(result)));
}
catch (RetriableTaskException retriableEx)
{
_logger.LogInformation(retriableEx, "Error in data processing task.");
await CleanResourceForFailureAsync(retriableEx);
throw;
}
catch (Exception ex)
{
_logger.LogInformation(ex, "Critical error in data processing task.");
await CleanResourceForFailureAsync(ex);
throw new RetriableTaskException(ex.Message);
}
finally
{
if (!_cancellationTokenSource.IsCancellationRequested)
{
_cancellationTokenSource.Cancel();
}
}
}