public async Task GivenImportTaskInput_WhenExceptionThrowForCleanData_ThenRetriableExceptionShouldBeThrow()
{
ImportProcessingTaskInputData inputData = GetInputData();
ImportProcessingProgress progress = new ImportProcessingProgress();
IImportResourceLoader loader = Substitute.For <IImportResourceLoader>();
IResourceBulkImporter importer = Substitute.For <IResourceBulkImporter>();
IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>();
IImportErrorStoreFactory importErrorStoreFactory = Substitute.For <IImportErrorStoreFactory>();
IContextUpdater contextUpdater = Substitute.For <IContextUpdater>();
RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
ILoggerFactory loggerFactory = new NullLoggerFactory();
importer.CleanResourceAsync(Arg.Any <ImportProcessingTaskInputData>(), Arg.Any <ImportProcessingProgress>(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
throw new InvalidOperationException();
});
progress.NeedCleanData = true;
ImportProcessingTask task = new ImportProcessingTask(
inputData,
progress,
loader,
importer,
importErrorStoreFactory,
contextUpdater,
contextAccessor,
loggerFactory);
await Assert.ThrowsAsync <RetriableTaskException>(() => task.ExecuteAsync());
}
public async Task GivenImportTaskInput_WhenOperationWasCancelledExceptionThrow_ThenTaskShouldBeCanceled()
{
ImportProcessingTaskInputData inputData = GetInputData();
ImportProcessingProgress progress = new ImportProcessingProgress();
IImportResourceLoader loader = Substitute.For <IImportResourceLoader>();
IResourceBulkImporter importer = Substitute.For <IResourceBulkImporter>();
IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>();
IImportErrorStoreFactory importErrorStoreFactory = Substitute.For <IImportErrorStoreFactory>();
IContextUpdater contextUpdater = Substitute.For <IContextUpdater>();
RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
ILoggerFactory loggerFactory = new NullLoggerFactory();
importer.Import(Arg.Any <Channel <ImportResource> >(), Arg.Any <IImportErrorStore>(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
throw new OperationCanceledException();
});
ImportProcessingTask task = new ImportProcessingTask(
inputData,
progress,
loader,
importer,
importErrorStoreFactory,
contextUpdater,
contextAccessor,
loggerFactory);
TaskResultData result = await task.ExecuteAsync();
Assert.Equal(TaskResult.Canceled, result.Result);
}
public ImportProcessingTask(
ImportProcessingTaskInputData inputData,
ImportProcessingProgress importProgress,
IImportResourceLoader importResourceLoader,
IResourceBulkImporter resourceBulkImporter,
IImportErrorStoreFactory importErrorStoreFactory,
IContextUpdater contextUpdater,
RequestContextAccessor <IFhirRequestContext> contextAccessor,
ILoggerFactory loggerFactory)
{
EnsureArg.IsNotNull(inputData, nameof(inputData));
EnsureArg.IsNotNull(importProgress, nameof(importProgress));
EnsureArg.IsNotNull(importResourceLoader, nameof(importResourceLoader));
EnsureArg.IsNotNull(resourceBulkImporter, nameof(resourceBulkImporter));
EnsureArg.IsNotNull(importErrorStoreFactory, nameof(importErrorStoreFactory));
EnsureArg.IsNotNull(contextUpdater, nameof(contextUpdater));
EnsureArg.IsNotNull(contextAccessor, nameof(contextAccessor));
EnsureArg.IsNotNull(loggerFactory, nameof(loggerFactory));
_inputData = inputData;
_importProgress = importProgress;
_importResourceLoader = importResourceLoader;
_resourceBulkImporter = resourceBulkImporter;
_importErrorStoreFactory = importErrorStoreFactory;
_contextUpdater = contextUpdater;
_contextAccessor = contextAccessor;
_logger = loggerFactory.CreateLogger <ImportProcessingTask>();
}
public async Task GivenAnOrchestratorTaskAndWrongEtag_WhenOrchestratorTaskStart_ThenTaskShouldFailedWithDetails()
{
IImportOrchestratorTaskDataStoreOperation fhirDataBulkImportOperation = Substitute.For <IImportOrchestratorTaskDataStoreOperation>();
IContextUpdater contextUpdater = Substitute.For <IContextUpdater>();
RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
ILoggerFactory loggerFactory = new NullLoggerFactory();
IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>();
ISequenceIdGenerator <long> sequenceIdGenerator = Substitute.For <ISequenceIdGenerator <long> >();
ImportOrchestratorTaskInputData importOrchestratorTaskInputData = new ImportOrchestratorTaskInputData();
ImportOrchestratorTaskContext importOrchestratorTaskContext = new ImportOrchestratorTaskContext();
ITaskManager taskManager = Substitute.For <ITaskManager>();
importOrchestratorTaskInputData.TaskId = Guid.NewGuid().ToString("N");
importOrchestratorTaskInputData.TaskCreateTime = Clock.UtcNow;
importOrchestratorTaskInputData.BaseUri = new Uri("http://dummy");
var inputs = new List <InputResource>();
inputs.Add(new InputResource()
{
Type = "Resource", Url = new Uri("http://dummy"), Etag = "dummy"
});
importOrchestratorTaskInputData.Input = inputs;
importOrchestratorTaskInputData.InputFormat = "ndjson";
importOrchestratorTaskInputData.InputSource = new Uri("http://dummy");
importOrchestratorTaskInputData.MaxConcurrentProcessingTaskCount = 1;
importOrchestratorTaskInputData.ProcessingTaskQueueId = "default";
importOrchestratorTaskInputData.RequestUri = new Uri("http://dummy");
integrationDataStoreClient.GetPropertiesAsync(Arg.Any <Uri>(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
Dictionary <string, object> properties = new Dictionary <string, object>();
properties[IntegrationDataStoreClientConstants.BlobPropertyETag] = "test";
properties[IntegrationDataStoreClientConstants.BlobPropertyLength] = 1000L;
return(properties);
});
sequenceIdGenerator.GetCurrentSequenceId().Returns(_ => 0L);
ImportOrchestratorTask orchestratorTask = new ImportOrchestratorTask(
importOrchestratorTaskInputData,
importOrchestratorTaskContext,
taskManager,
sequenceIdGenerator,
contextUpdater,
contextAccessor,
fhirDataBulkImportOperation,
integrationDataStoreClient,
loggerFactory);
TaskResultData result = await orchestratorTask.ExecuteAsync();
ImportTaskErrorResult resultDetails = JsonConvert.DeserializeObject <ImportTaskErrorResult>(result.ResultData);
Assert.Equal(TaskResult.Fail, result.Result);
Assert.Equal(HttpStatusCode.BadRequest, resultDetails.HttpStatusCode);
Assert.NotEmpty(resultDetails.ErrorMessage);
}
// TODO: API to remove or overwrite
public bool Register(string commandName, IContextUpdater updater)
{
var key = GetKey(commandName);
if (!updaters_.ContainsKey(key))
{
updaters_.Add(key, updater);
return(true);
}
return(false);
}
public async Task GivenImportTaskInput_WhenExceptionThrowForLoad_ThenRetriableExceptionShouldBeThrow()
{
ImportProcessingTaskInputData inputData = GetInputData();
ImportProcessingProgress progress = new ImportProcessingProgress();
IImportResourceLoader loader = Substitute.For <IImportResourceLoader>();
IResourceBulkImporter importer = Substitute.For <IResourceBulkImporter>();
IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>();
IImportErrorStoreFactory importErrorStoreFactory = Substitute.For <IImportErrorStoreFactory>();
IContextUpdater contextUpdater = Substitute.For <IContextUpdater>();
RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
ILoggerFactory loggerFactory = new NullLoggerFactory();
loader.LoadResources(Arg.Any <string>(), Arg.Any <long>(), Arg.Any <string>(), Arg.Any <Func <long, long> >(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
long startIndex = (long)callInfo[1];
Func <long, long> idGenerator = (Func <long, long>)callInfo[3];
Channel <ImportResource> resourceChannel = Channel.CreateUnbounded <ImportResource>();
Task loadTask = Task.Run(() =>
{
try
{
throw new InvalidOperationException();
}
finally
{
resourceChannel.Writer.Complete();
}
});
return(resourceChannel, loadTask);
});
ImportProcessingTask task = new ImportProcessingTask(
inputData,
progress,
loader,
importer,
importErrorStoreFactory,
contextUpdater,
contextAccessor,
loggerFactory);
await Assert.ThrowsAsync <RetriableTaskException>(() => task.ExecuteAsync());
}
public ITask Create(TaskInfo taskInfo)
{
EnsureArg.IsNotNull(taskInfo, nameof(taskInfo));
if (taskInfo.TaskTypeId == ImportProcessingTask.ImportProcessingTaskId)
{
IContextUpdater contextUpdater = _contextUpdaterFactory.CreateContextUpdater(taskInfo.TaskId, taskInfo.RunId);
ImportProcessingTaskInputData inputData = JsonConvert.DeserializeObject <ImportProcessingTaskInputData>(taskInfo.InputData);
ImportProcessingProgress importProgress = string.IsNullOrEmpty(taskInfo.Context) ? new ImportProcessingProgress() : JsonConvert.DeserializeObject <ImportProcessingProgress>(taskInfo.Context);
return(new ImportProcessingTask(
inputData,
importProgress,
_importResourceLoader,
_resourceBulkImporter,
_importErrorStoreFactory,
contextUpdater,
_contextAccessor,
_loggerFactory));
}
if (taskInfo.TaskTypeId == ImportOrchestratorTask.ImportOrchestratorTaskId)
{
IContextUpdater contextUpdater = _contextUpdaterFactory.CreateContextUpdater(taskInfo.TaskId, taskInfo.RunId);
ImportOrchestratorTaskInputData inputData = JsonConvert.DeserializeObject <ImportOrchestratorTaskInputData>(taskInfo.InputData);
ImportOrchestratorTaskContext orchestratorTaskProgress = string.IsNullOrEmpty(taskInfo.Context) ? new ImportOrchestratorTaskContext() : JsonConvert.DeserializeObject <ImportOrchestratorTaskContext>(taskInfo.Context);
return(new ImportOrchestratorTask(
_mediator,
inputData,
orchestratorTaskProgress,
_taskmanager,
_sequenceIdGenerator,
contextUpdater,
_contextAccessor,
_importOrchestratorTaskDataStoreOperation,
_integrationDataStoreClient,
_loggerFactory));
}
return(null);
}
public async Task GivenImportTaskInput_WhenExceptionThrowForImport_ThenContextShouldBeUpdatedBeforeFailure()
{
long currentIndex = 100;
ImportProcessingTaskInputData inputData = GetInputData();
ImportProcessingProgress progress = new ImportProcessingProgress();
IImportResourceLoader loader = Substitute.For <IImportResourceLoader>();
IResourceBulkImporter importer = Substitute.For <IResourceBulkImporter>();
IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>();
IImportErrorStoreFactory importErrorStoreFactory = Substitute.For <IImportErrorStoreFactory>();
IContextUpdater contextUpdater = Substitute.For <IContextUpdater>();
RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
ILoggerFactory loggerFactory = new NullLoggerFactory();
loader.LoadResources(Arg.Any <string>(), Arg.Any <long>(), Arg.Any <string>(), Arg.Any <Func <long, long> >(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
long startIndex = (long)callInfo[1];
Func <long, long> idGenerator = (Func <long, long>)callInfo[3];
Channel <ImportResource> resourceChannel = Channel.CreateUnbounded <ImportResource>();
resourceChannel.Writer.Complete();
return(resourceChannel, Task.CompletedTask);
});
importer.Import(Arg.Any <Channel <ImportResource> >(), Arg.Any <IImportErrorStore>(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
Channel <ImportProcessingProgress> progressChannel = Channel.CreateUnbounded <ImportProcessingProgress>();
Task loadTask = Task.Run(async() =>
{
try
{
ImportProcessingProgress progress = new ImportProcessingProgress();
progress.CurrentIndex = currentIndex;
await progressChannel.Writer.WriteAsync(progress);
throw new InvalidOperationException();
}
finally
{
progressChannel.Writer.Complete();
}
});
return(progressChannel, loadTask);
});
string context = null;
contextUpdater.UpdateContextAsync(Arg.Any <string>(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
context = (string)callInfo[0];
return(Task.CompletedTask);
});
ImportProcessingTask task = new ImportProcessingTask(
inputData,
progress,
loader,
importer,
importErrorStoreFactory,
contextUpdater,
contextAccessor,
loggerFactory);
await Assert.ThrowsAsync <RetriableTaskException>(() => task.ExecuteAsync());
ImportProcessingProgress progressForContext = JsonConvert.DeserializeObject <ImportProcessingProgress>(context);
Assert.Equal(progressForContext.CurrentIndex, currentIndex);
}
private static async Task VerifyCommonImportTaskAsync(ImportProcessingTaskInputData inputData, ImportProcessingProgress progress)
{
long startIndexFromProgress = progress.CurrentIndex;
long succeedCountFromProgress = progress.SucceedImportCount;
long failedCountFromProgress = progress.FailedImportCount;
IImportResourceLoader loader = Substitute.For <IImportResourceLoader>();
IResourceBulkImporter importer = Substitute.For <IResourceBulkImporter>();
IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>();
IImportErrorStoreFactory importErrorStoreFactory = Substitute.For <IImportErrorStoreFactory>();
IContextUpdater contextUpdater = Substitute.For <IContextUpdater>();
RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
ILoggerFactory loggerFactory = new NullLoggerFactory();
long cleanStart = -1;
long cleanEnd = -1;
importer.CleanResourceAsync(Arg.Any <ImportProcessingTaskInputData>(), Arg.Any <ImportProcessingProgress>(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
var inputData = (ImportProcessingTaskInputData)callInfo[0];
var progress = (ImportProcessingProgress)callInfo[1];
long beginSequenceId = inputData.BeginSequenceId;
long endSequenceId = inputData.EndSequenceId;
long endIndex = progress.CurrentIndex;
cleanStart = beginSequenceId + endIndex;
cleanEnd = endSequenceId;
return(Task.CompletedTask);
});
loader.LoadResources(Arg.Any <string>(), Arg.Any <long>(), Arg.Any <string>(), Arg.Any <Func <long, long> >(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
long startIndex = (long)callInfo[1];
Func <long, long> idGenerator = (Func <long, long>)callInfo[3];
Channel <ImportResource> resourceChannel = Channel.CreateUnbounded <ImportResource>();
Task loadTask = Task.Run(async() =>
{
ResourceWrapper resourceWrapper = new ResourceWrapper(
Guid.NewGuid().ToString(),
"0",
"Dummy",
new RawResource(Guid.NewGuid().ToString(), Fhir.Core.Models.FhirResourceFormat.Json, true),
new ResourceRequest("POST"),
DateTimeOffset.UtcNow,
false,
null,
null,
null,
"SearchParam");
await resourceChannel.Writer.WriteAsync(new ImportResource(idGenerator(startIndex), startIndex, resourceWrapper));
await resourceChannel.Writer.WriteAsync(new ImportResource(idGenerator(startIndex + 1), startIndex + 1, "Error"));
resourceChannel.Writer.Complete();
});
return(resourceChannel, loadTask);
});
importer.Import(Arg.Any <Channel <ImportResource> >(), Arg.Any <IImportErrorStore>(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
Channel <ImportResource> resourceChannel = (Channel <ImportResource>)callInfo[0];
Channel <ImportProcessingProgress> progressChannel = Channel.CreateUnbounded <ImportProcessingProgress>();
Task loadTask = Task.Run(async() =>
{
ImportProcessingProgress progress = new ImportProcessingProgress();
await foreach (ImportResource resource in resourceChannel.Reader.ReadAllAsync())
{
if (string.IsNullOrEmpty(resource.ImportError))
{
progress.SucceedImportCount++;
}
else
{
progress.FailedImportCount++;
}
progress.CurrentIndex = resource.Index + 1;
}
await progressChannel.Writer.WriteAsync(progress);
progressChannel.Writer.Complete();
});
return(progressChannel, loadTask);
});
string context = null;
contextUpdater.UpdateContextAsync(Arg.Any <string>(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
context = (string)callInfo[0];
return(Task.CompletedTask);
});
progress.NeedCleanData = true;
ImportProcessingTask task = new ImportProcessingTask(
inputData,
progress,
loader,
importer,
importErrorStoreFactory,
contextUpdater,
contextAccessor,
loggerFactory);
TaskResultData taskResult = await task.ExecuteAsync();
Assert.Equal(TaskResult.Success, taskResult.Result);
ImportProcessingTaskResult result = JsonConvert.DeserializeObject <ImportProcessingTaskResult>(taskResult.ResultData);
Assert.Equal(1 + failedCountFromProgress, result.FailedCount);
Assert.Equal(1 + succeedCountFromProgress, result.SucceedCount);
ImportProcessingProgress progressForContext = JsonConvert.DeserializeObject <ImportProcessingProgress>(context);
Assert.Equal(progressForContext.SucceedImportCount, result.SucceedCount);
Assert.Equal(progressForContext.FailedImportCount, result.FailedCount);
Assert.Equal(startIndexFromProgress + 2, progressForContext.CurrentIndex);
Assert.Equal(startIndexFromProgress, cleanStart);
Assert.Equal(inputData.EndSequenceId, cleanEnd);
}
private static async Task VerifyCommonOrchestratorTaskAsync(int inputFileCount, int concurrentCount, int resumeFrom = -1)
{
IImportOrchestratorTaskDataStoreOperation fhirDataBulkImportOperation = Substitute.For <IImportOrchestratorTaskDataStoreOperation>();
IContextUpdater contextUpdater = Substitute.For <IContextUpdater>();
RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
ILoggerFactory loggerFactory = new NullLoggerFactory();
IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>();
ISequenceIdGenerator <long> sequenceIdGenerator = Substitute.For <ISequenceIdGenerator <long> >();
IMediator mediator = Substitute.For <IMediator>();
ImportOrchestratorTaskInputData importOrchestratorTaskInputData = new ImportOrchestratorTaskInputData();
ImportOrchestratorTaskContext importOrchestratorTaskContext = new ImportOrchestratorTaskContext();
List <(long begin, long end)> surrogatedIdRanges = new List <(long begin, long end)>();
TestTaskManager taskManager = new TestTaskManager(t =>
{
if (t == null)
{
return(null);
}
if (t.Status == TaskManagement.TaskStatus.Completed)
{
return(t);
}
ImportProcessingTaskInputData processingInput = JsonConvert.DeserializeObject <ImportProcessingTaskInputData>(t.InputData);
ImportProcessingTaskResult processingResult = new ImportProcessingTaskResult();
processingResult.ResourceType = processingInput.ResourceType;
processingResult.SucceedCount = 1;
processingResult.FailedCount = 1;
processingResult.ErrorLogLocation = "http://dummy/error";
surrogatedIdRanges.Add((processingInput.BeginSequenceId, processingInput.EndSequenceId));
t.Result = JsonConvert.SerializeObject(new TaskResultData(TaskResult.Success, JsonConvert.SerializeObject(processingResult)));
t.Status = TaskManagement.TaskStatus.Completed;
return(t);
});
importOrchestratorTaskInputData.TaskId = Guid.NewGuid().ToString("N");
importOrchestratorTaskInputData.TaskCreateTime = Clock.UtcNow;
importOrchestratorTaskInputData.BaseUri = new Uri("http://dummy");
var inputs = new List <InputResource>();
bool resumeMode = resumeFrom >= 0;
for (int i = 0; i < inputFileCount; ++i)
{
string location = $"http://dummy/{i}";
inputs.Add(new InputResource()
{
Type = "Resource", Url = new Uri(location)
});
if (resumeMode)
{
if (i <= resumeFrom)
{
TaskInfo taskInfo = new TaskInfo();
taskInfo.TaskId = Guid.NewGuid().ToString("N");
ImportProcessingTaskResult processingResult = new ImportProcessingTaskResult();
processingResult.ResourceType = "Resource";
processingResult.SucceedCount = 1;
processingResult.FailedCount = 1;
processingResult.ErrorLogLocation = "http://dummy/error";
taskInfo.Result = JsonConvert.SerializeObject(new TaskResultData(TaskResult.Success, JsonConvert.SerializeObject(processingResult)));
taskInfo.Status = TaskManagement.TaskStatus.Completed;
await taskManager.CreateTaskAsync(taskInfo, false, CancellationToken.None);
importOrchestratorTaskContext.DataProcessingTasks[new Uri(location)] = taskInfo;
}
else
{
TaskInfo taskInfo = new TaskInfo();
taskInfo.TaskId = Guid.NewGuid().ToString("N");
ImportProcessingTaskInputData processingInput = new ImportProcessingTaskInputData();
processingInput.BaseUriString = "http://dummy";
processingInput.BeginSequenceId = i;
processingInput.EndSequenceId = i + 1;
processingInput.ResourceType = "Resource";
taskInfo.InputData = JsonConvert.SerializeObject(processingInput);
await taskManager.CreateTaskAsync(taskInfo, false, CancellationToken.None);
importOrchestratorTaskContext.DataProcessingTasks[new Uri(location)] = taskInfo;
}
importOrchestratorTaskContext.Progress = ImportOrchestratorTaskProgress.SubTaskRecordsGenerated;
}
}
importOrchestratorTaskInputData.Input = inputs;
importOrchestratorTaskInputData.InputFormat = "ndjson";
importOrchestratorTaskInputData.InputSource = new Uri("http://dummy");
importOrchestratorTaskInputData.MaxConcurrentProcessingTaskCount = concurrentCount;
importOrchestratorTaskInputData.ProcessingTaskQueueId = "default";
importOrchestratorTaskInputData.RequestUri = new Uri("http://dummy");
integrationDataStoreClient.GetPropertiesAsync(Arg.Any <Uri>(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
Dictionary <string, object> properties = new Dictionary <string, object>();
properties[IntegrationDataStoreClientConstants.BlobPropertyETag] = "test";
properties[IntegrationDataStoreClientConstants.BlobPropertyLength] = 1000L;
return(properties);
});
sequenceIdGenerator.GetCurrentSequenceId().Returns(_ => 0L);
ImportOrchestratorTask orchestratorTask = new ImportOrchestratorTask(
mediator,
importOrchestratorTaskInputData,
importOrchestratorTaskContext,
taskManager,
sequenceIdGenerator,
contextUpdater,
contextAccessor,
fhirDataBulkImportOperation,
integrationDataStoreClient,
loggerFactory);
orchestratorTask.PollingFrequencyInSeconds = 0;
TaskResultData result = await orchestratorTask.ExecuteAsync();
ImportTaskResult resultDetails = JsonConvert.DeserializeObject <ImportTaskResult>(result.ResultData);
Assert.Equal(TaskResult.Success, result.Result);
Assert.Equal(inputFileCount, resultDetails.Output.Count);
foreach (ImportOperationOutcome outcome in resultDetails.Output)
{
Assert.Equal(1, outcome.Count);
Assert.NotNull(outcome.InputUrl);
Assert.NotEmpty(outcome.Type);
}
Assert.Equal(inputFileCount, resultDetails.Error.Count);
foreach (ImportFailedOperationOutcome outcome in resultDetails.Error)
{
Assert.Equal(1, outcome.Count);
Assert.NotNull(outcome.InputUrl);
Assert.NotEmpty(outcome.Type);
Assert.NotEmpty(outcome.Url);
}
Assert.NotEmpty(resultDetails.Request);
Assert.Equal(importOrchestratorTaskInputData.TaskCreateTime, resultDetails.TransactionTime);
var orderedSurrogatedIdRanges = surrogatedIdRanges.OrderBy(r => r.begin).ToArray();
Assert.Equal(inputFileCount, orderedSurrogatedIdRanges.Length + resumeFrom + 1);
for (int i = 0; i < orderedSurrogatedIdRanges.Length - 1; ++i)
{
Assert.True(orderedSurrogatedIdRanges[i].end > orderedSurrogatedIdRanges[i].begin);
Assert.True(orderedSurrogatedIdRanges[i].end <= orderedSurrogatedIdRanges[i + 1].begin);
}
_ = mediator.Received().Publish(
Arg.Is <ImportTaskMetricsNotification>(
notification => notification.Id == importOrchestratorTaskInputData.TaskId &&
notification.Status == TaskResult.Success.ToString() &&
notification.CreatedTime == importOrchestratorTaskInputData.TaskCreateTime &&
notification.SucceedCount == resultDetails.Output.Sum(o => o.Count) &&
notification.FailedCount == resultDetails.Error.Sum(e => e.Count)),
Arg.Any <CancellationToken>());
}
public async Task GivenAnOrchestratorTask_WhenCancelBefore_ThenCanceledResultShouldBeReturn()
{
IImportOrchestratorTaskDataStoreOperation fhirDataBulkImportOperation = Substitute.For <IImportOrchestratorTaskDataStoreOperation>();
IContextUpdater contextUpdater = Substitute.For <IContextUpdater>();
RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
ILoggerFactory loggerFactory = new NullLoggerFactory();
IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>();
ISequenceIdGenerator <long> sequenceIdGenerator = Substitute.For <ISequenceIdGenerator <long> >();
IMediator mediator = Substitute.For <IMediator>();
ImportOrchestratorTaskInputData importOrchestratorTaskInputData = new ImportOrchestratorTaskInputData();
ImportOrchestratorTaskContext importOrchestratorTaskContext = new ImportOrchestratorTaskContext();
List <(long begin, long end)> surrogatedIdRanges = new List <(long begin, long end)>();
TestTaskManager taskManager = new TestTaskManager(t =>
{
if (t == null)
{
return(null);
}
ImportProcessingTaskInputData processingInput = JsonConvert.DeserializeObject <ImportProcessingTaskInputData>(t.InputData);
ImportProcessingTaskResult processingResult = new ImportProcessingTaskResult();
processingResult.ResourceType = processingInput.ResourceType;
processingResult.SucceedCount = 1;
processingResult.FailedCount = 1;
processingResult.ErrorLogLocation = "http://dummy/error";
surrogatedIdRanges.Add((processingInput.BeginSequenceId, processingInput.EndSequenceId));
t.Result = JsonConvert.SerializeObject(new TaskResultData(TaskResult.Success, JsonConvert.SerializeObject(processingResult)));
t.Status = TaskManagement.TaskStatus.Completed;
return(t);
});
importOrchestratorTaskInputData.TaskId = Guid.NewGuid().ToString("N");
importOrchestratorTaskInputData.TaskCreateTime = Clock.UtcNow;
importOrchestratorTaskInputData.BaseUri = new Uri("http://dummy");
var inputs = new List <InputResource>();
inputs.Add(new InputResource()
{
Type = "Resource", Url = new Uri($"http://dummy")
});
importOrchestratorTaskInputData.Input = inputs;
importOrchestratorTaskInputData.InputFormat = "ndjson";
importOrchestratorTaskInputData.InputSource = new Uri("http://dummy");
importOrchestratorTaskInputData.MaxConcurrentProcessingTaskCount = 1;
importOrchestratorTaskInputData.ProcessingTaskQueueId = "default";
importOrchestratorTaskInputData.RequestUri = new Uri("http://dummy");
integrationDataStoreClient.GetPropertiesAsync(Arg.Any <Uri>(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
Dictionary <string, object> properties = new Dictionary <string, object>();
properties[IntegrationDataStoreClientConstants.BlobPropertyETag] = "test";
properties[IntegrationDataStoreClientConstants.BlobPropertyLength] = 1000L;
return(properties);
});
string latestContext = null;
contextUpdater.UpdateContextAsync(Arg.Any <string>(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
latestContext = (string)callInfo[0];
return(Task.CompletedTask);
});
sequenceIdGenerator.GetCurrentSequenceId().Returns <long>(_ => 0L);
ImportOrchestratorTask orchestratorTask = new ImportOrchestratorTask(
mediator,
importOrchestratorTaskInputData,
importOrchestratorTaskContext,
taskManager,
sequenceIdGenerator,
contextUpdater,
contextAccessor,
fhirDataBulkImportOperation,
integrationDataStoreClient,
loggerFactory);
orchestratorTask.PollingFrequencyInSeconds = 0;
orchestratorTask.Cancel();
TaskResultData taskResult = await orchestratorTask.ExecuteAsync();
Assert.Equal(TaskResult.Canceled, taskResult.Result);
_ = mediator.Received().Publish(
Arg.Is <ImportTaskMetricsNotification>(
notification => notification.Id == importOrchestratorTaskInputData.TaskId &&
notification.Status == TaskResult.Canceled.ToString() &&
notification.CreatedTime == importOrchestratorTaskInputData.TaskCreateTime &&
notification.SucceedCount == null &&
notification.FailedCount == null),
Arg.Any <CancellationToken>());
}
public async Task GivenAnOrchestratorTask_WhenIntegrationExceptionThrow_ThenTaskShouldFailedWithDetails()
{
IImportOrchestratorTaskDataStoreOperation fhirDataBulkImportOperation = Substitute.For <IImportOrchestratorTaskDataStoreOperation>();
IContextUpdater contextUpdater = Substitute.For <IContextUpdater>();
RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
ILoggerFactory loggerFactory = new NullLoggerFactory();
IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>();
ISequenceIdGenerator <long> sequenceIdGenerator = Substitute.For <ISequenceIdGenerator <long> >();
ImportOrchestratorTaskInputData importOrchestratorTaskInputData = new ImportOrchestratorTaskInputData();
ImportOrchestratorTaskContext importOrchestratorTaskContext = new ImportOrchestratorTaskContext();
ITaskManager taskManager = Substitute.For <ITaskManager>();
IMediator mediator = Substitute.For <IMediator>();
importOrchestratorTaskInputData.TaskId = Guid.NewGuid().ToString("N");
importOrchestratorTaskInputData.TaskCreateTime = Clock.UtcNow;
importOrchestratorTaskInputData.BaseUri = new Uri("http://dummy");
var inputs = new List <InputResource>();
inputs.Add(new InputResource()
{
Type = "Resource", Url = new Uri("http://dummy"), Etag = "dummy"
});
importOrchestratorTaskInputData.Input = inputs;
importOrchestratorTaskInputData.InputFormat = "ndjson";
importOrchestratorTaskInputData.InputSource = new Uri("http://dummy");
importOrchestratorTaskInputData.MaxConcurrentProcessingTaskCount = 1;
importOrchestratorTaskInputData.ProcessingTaskQueueId = "default";
importOrchestratorTaskInputData.RequestUri = new Uri("http://dummy");
integrationDataStoreClient.GetPropertiesAsync(Arg.Any <Uri>(), Arg.Any <CancellationToken>())
.Returns <Task <Dictionary <string, object> > >(_ =>
{
throw new IntegrationDataStoreException("dummy", HttpStatusCode.Unauthorized);
});
sequenceIdGenerator.GetCurrentSequenceId().Returns(_ => 0L);
ImportOrchestratorTask orchestratorTask = new ImportOrchestratorTask(
mediator,
importOrchestratorTaskInputData,
importOrchestratorTaskContext,
taskManager,
sequenceIdGenerator,
contextUpdater,
contextAccessor,
fhirDataBulkImportOperation,
integrationDataStoreClient,
loggerFactory);
TaskResultData result = await orchestratorTask.ExecuteAsync();
ImportTaskErrorResult resultDetails = JsonConvert.DeserializeObject <ImportTaskErrorResult>(result.ResultData);
Assert.Equal(TaskResult.Fail, result.Result);
Assert.Equal(HttpStatusCode.Unauthorized, resultDetails.HttpStatusCode);
Assert.NotEmpty(resultDetails.ErrorMessage);
_ = mediator.Received().Publish(
Arg.Is <ImportTaskMetricsNotification>(
notification => notification.Id == importOrchestratorTaskInputData.TaskId &&
notification.Status == TaskResult.Fail.ToString() &&
notification.CreatedTime == importOrchestratorTaskInputData.TaskCreateTime &&
notification.DataSize == null &&
notification.SucceedCount == null &&
notification.FailedCount == null),
Arg.Any <CancellationToken>());
}
public async Task GivenAnOrchestratorTask_WhenSubTaskFailed_ThenImportProcessingExceptionShouldBeThrowAndContextUpdated()
{
IImportOrchestratorTaskDataStoreOperation fhirDataBulkImportOperation = Substitute.For <IImportOrchestratorTaskDataStoreOperation>();
IContextUpdater contextUpdater = Substitute.For <IContextUpdater>();
RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
ILoggerFactory loggerFactory = new NullLoggerFactory();
IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>();
ISequenceIdGenerator <long> sequenceIdGenerator = Substitute.For <ISequenceIdGenerator <long> >();
ImportOrchestratorTaskInputData importOrchestratorTaskInputData = new ImportOrchestratorTaskInputData();
ImportOrchestratorTaskContext importOrchestratorTaskContext = new ImportOrchestratorTaskContext();
List <(long begin, long end)> surrogatedIdRanges = new List <(long begin, long end)>();
TestTaskManager taskManager = new TestTaskManager(t =>
{
if (t == null)
{
return(null);
}
TaskResultData resultData = new TaskResultData();
resultData.Result = TaskResult.Fail;
resultData.ResultData = "error";
t.Result = JsonConvert.SerializeObject(resultData);
t.Status = TaskManagement.TaskStatus.Completed;
return(t);
});
importOrchestratorTaskInputData.TaskId = Guid.NewGuid().ToString("N");
importOrchestratorTaskInputData.TaskCreateTime = Clock.UtcNow;
importOrchestratorTaskInputData.BaseUri = new Uri("http://dummy");
var inputs = new List <InputResource>();
inputs.Add(new InputResource()
{
Type = "Resource", Url = new Uri($"http://dummy")
});
importOrchestratorTaskInputData.Input = inputs;
importOrchestratorTaskInputData.InputFormat = "ndjson";
importOrchestratorTaskInputData.InputSource = new Uri("http://dummy");
importOrchestratorTaskInputData.MaxConcurrentProcessingTaskCount = 1;
importOrchestratorTaskInputData.ProcessingTaskQueueId = "default";
importOrchestratorTaskInputData.RequestUri = new Uri("http://dummy");
integrationDataStoreClient.GetPropertiesAsync(Arg.Any <Uri>(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
Dictionary <string, object> properties = new Dictionary <string, object>();
properties[IntegrationDataStoreClientConstants.BlobPropertyETag] = "test";
properties[IntegrationDataStoreClientConstants.BlobPropertyLength] = 1000L;
return(properties);
});
string latestContext = null;
contextUpdater.UpdateContextAsync(Arg.Any <string>(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
latestContext = (string)callInfo[0];
return(Task.CompletedTask);
});
sequenceIdGenerator.GetCurrentSequenceId().Returns <long>(_ => 0L);
ImportOrchestratorTask orchestratorTask = new ImportOrchestratorTask(
importOrchestratorTaskInputData,
importOrchestratorTaskContext,
taskManager,
sequenceIdGenerator,
contextUpdater,
contextAccessor,
fhirDataBulkImportOperation,
integrationDataStoreClient,
loggerFactory);
orchestratorTask.PollingFrequencyInSeconds = 0;
TaskResultData taskResultData = await orchestratorTask.ExecuteAsync();
Assert.Equal(TaskResult.Fail, taskResultData.Result);
ImportOrchestratorTaskContext context = JsonConvert.DeserializeObject <ImportOrchestratorTaskContext>(latestContext);
Assert.Equal(ImportOrchestratorTaskProgress.SubTaskRecordsGenerated, context.Progress);
}
