public ITask Create(TaskInfo taskInfo) { EnsureArg.IsNotNull(taskInfo, nameof(taskInfo)); if (taskInfo.TaskTypeId == ImportProcessingTask.ImportProcessingTaskId) { IContextUpdater contextUpdater = _contextUpdaterFactory.CreateContextUpdater(taskInfo.TaskId, taskInfo.RunId); ImportProcessingTaskInputData inputData = JsonConvert.DeserializeObject <ImportProcessingTaskInputData>(taskInfo.InputData); ImportProcessingProgress importProgress = string.IsNullOrEmpty(taskInfo.Context) ? new ImportProcessingProgress() : JsonConvert.DeserializeObject <ImportProcessingProgress>(taskInfo.Context); return(new ImportProcessingTask( inputData, importProgress, _importResourceLoader, _resourceBulkImporter, _importErrorStoreFactory, contextUpdater, _contextAccessor, _loggerFactory)); } if (taskInfo.TaskTypeId == ImportOrchestratorTask.ImportOrchestratorTaskId) { IContextUpdater contextUpdater = _contextUpdaterFactory.CreateContextUpdater(taskInfo.TaskId, taskInfo.RunId); ImportOrchestratorTaskInputData inputData = JsonConvert.DeserializeObject <ImportOrchestratorTaskInputData>(taskInfo.InputData); ImportOrchestratorTaskContext orchestratorTaskProgress = string.IsNullOrEmpty(taskInfo.Context) ? new ImportOrchestratorTaskContext() : JsonConvert.DeserializeObject <ImportOrchestratorTaskContext>(taskInfo.Context); return(new ImportOrchestratorTask( _mediator, inputData, orchestratorTaskProgress, _taskmanager, _sequenceIdGenerator, contextUpdater, _contextAccessor, _importOrchestratorTaskDataStoreOperation, _integrationDataStoreClient, _loggerFactory)); } return(null); }
private static async Task VerifyCommonOrchestratorTaskAsync(int inputFileCount, int concurrentCount, int resumeFrom = -1) { IImportOrchestratorTaskDataStoreOperation fhirDataBulkImportOperation = Substitute.For <IImportOrchestratorTaskDataStoreOperation>(); IContextUpdater contextUpdater = Substitute.For <IContextUpdater>(); RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >(); ILoggerFactory loggerFactory = new NullLoggerFactory(); IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>(); ISequenceIdGenerator <long> sequenceIdGenerator = Substitute.For <ISequenceIdGenerator <long> >(); IMediator mediator = Substitute.For <IMediator>(); ImportOrchestratorTaskInputData importOrchestratorTaskInputData = new ImportOrchestratorTaskInputData(); ImportOrchestratorTaskContext importOrchestratorTaskContext = new ImportOrchestratorTaskContext(); List <(long begin, long end)> surrogatedIdRanges = new List <(long begin, long end)>(); TestTaskManager taskManager = new TestTaskManager(t => { if (t == null) { return(null); } if (t.Status == TaskManagement.TaskStatus.Completed) { return(t); } ImportProcessingTaskInputData processingInput = JsonConvert.DeserializeObject <ImportProcessingTaskInputData>(t.InputData); ImportProcessingTaskResult processingResult = new ImportProcessingTaskResult(); processingResult.ResourceType = processingInput.ResourceType; processingResult.SucceedCount = 1; processingResult.FailedCount = 1; processingResult.ErrorLogLocation = "http://dummy/error"; surrogatedIdRanges.Add((processingInput.BeginSequenceId, processingInput.EndSequenceId)); t.Result = JsonConvert.SerializeObject(new TaskResultData(TaskResult.Success, JsonConvert.SerializeObject(processingResult))); t.Status = TaskManagement.TaskStatus.Completed; return(t); }); importOrchestratorTaskInputData.TaskId = Guid.NewGuid().ToString("N"); importOrchestratorTaskInputData.TaskCreateTime = Clock.UtcNow; importOrchestratorTaskInputData.BaseUri = new Uri("http://dummy"); var inputs = new List <InputResource>(); bool resumeMode = resumeFrom >= 0; for (int i = 0; i < inputFileCount; ++i) { string location = $"http://dummy/{i}"; inputs.Add(new InputResource() { Type = "Resource", Url = new Uri(location) }); if (resumeMode) { if (i <= resumeFrom) { TaskInfo taskInfo = new TaskInfo(); taskInfo.TaskId = Guid.NewGuid().ToString("N"); ImportProcessingTaskResult processingResult = new ImportProcessingTaskResult(); processingResult.ResourceType = "Resource"; processingResult.SucceedCount = 1; processingResult.FailedCount = 1; processingResult.ErrorLogLocation = "http://dummy/error"; taskInfo.Result = JsonConvert.SerializeObject(new TaskResultData(TaskResult.Success, JsonConvert.SerializeObject(processingResult))); taskInfo.Status = TaskManagement.TaskStatus.Completed; await taskManager.CreateTaskAsync(taskInfo, false, CancellationToken.None); importOrchestratorTaskContext.DataProcessingTasks[new Uri(location)] = taskInfo; } else { TaskInfo taskInfo = new TaskInfo(); taskInfo.TaskId = Guid.NewGuid().ToString("N"); ImportProcessingTaskInputData processingInput = new ImportProcessingTaskInputData(); processingInput.BaseUriString = "http://dummy"; processingInput.BeginSequenceId = i; processingInput.EndSequenceId = i + 1; processingInput.ResourceType = "Resource"; taskInfo.InputData = JsonConvert.SerializeObject(processingInput); await taskManager.CreateTaskAsync(taskInfo, false, CancellationToken.None); importOrchestratorTaskContext.DataProcessingTasks[new Uri(location)] = taskInfo; } importOrchestratorTaskContext.Progress = ImportOrchestratorTaskProgress.SubTaskRecordsGenerated; } } importOrchestratorTaskInputData.Input = inputs; importOrchestratorTaskInputData.InputFormat = "ndjson"; importOrchestratorTaskInputData.InputSource = new Uri("http://dummy"); importOrchestratorTaskInputData.MaxConcurrentProcessingTaskCount = concurrentCount; importOrchestratorTaskInputData.ProcessingTaskQueueId = "default"; importOrchestratorTaskInputData.RequestUri = new Uri("http://dummy"); integrationDataStoreClient.GetPropertiesAsync(Arg.Any <Uri>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { Dictionary <string, object> properties = new Dictionary <string, object>(); properties[IntegrationDataStoreClientConstants.BlobPropertyETag] = "test"; properties[IntegrationDataStoreClientConstants.BlobPropertyLength] = 1000L; return(properties); }); sequenceIdGenerator.GetCurrentSequenceId().Returns(_ => 0L); ImportOrchestratorTask orchestratorTask = new ImportOrchestratorTask( mediator, importOrchestratorTaskInputData, importOrchestratorTaskContext, taskManager, sequenceIdGenerator, contextUpdater, contextAccessor, fhirDataBulkImportOperation, integrationDataStoreClient, loggerFactory); orchestratorTask.PollingFrequencyInSeconds = 0; TaskResultData result = await orchestratorTask.ExecuteAsync(); ImportTaskResult resultDetails = JsonConvert.DeserializeObject <ImportTaskResult>(result.ResultData); Assert.Equal(TaskResult.Success, result.Result); Assert.Equal(inputFileCount, resultDetails.Output.Count); foreach (ImportOperationOutcome outcome in resultDetails.Output) { Assert.Equal(1, outcome.Count); Assert.NotNull(outcome.InputUrl); Assert.NotEmpty(outcome.Type); } Assert.Equal(inputFileCount, resultDetails.Error.Count); foreach (ImportFailedOperationOutcome outcome in resultDetails.Error) { Assert.Equal(1, outcome.Count); Assert.NotNull(outcome.InputUrl); Assert.NotEmpty(outcome.Type); Assert.NotEmpty(outcome.Url); } Assert.NotEmpty(resultDetails.Request); Assert.Equal(importOrchestratorTaskInputData.TaskCreateTime, resultDetails.TransactionTime); var orderedSurrogatedIdRanges = surrogatedIdRanges.OrderBy(r => r.begin).ToArray(); Assert.Equal(inputFileCount, orderedSurrogatedIdRanges.Length + resumeFrom + 1); for (int i = 0; i < orderedSurrogatedIdRanges.Length - 1; ++i) { Assert.True(orderedSurrogatedIdRanges[i].end > orderedSurrogatedIdRanges[i].begin); Assert.True(orderedSurrogatedIdRanges[i].end <= orderedSurrogatedIdRanges[i + 1].begin); } _ = mediator.Received().Publish( Arg.Is <ImportTaskMetricsNotification>( notification => notification.Id == importOrchestratorTaskInputData.TaskId && notification.Status == TaskResult.Success.ToString() && notification.CreatedTime == importOrchestratorTaskInputData.TaskCreateTime && notification.SucceedCount == resultDetails.Output.Sum(o => o.Count) && notification.FailedCount == resultDetails.Error.Sum(e => e.Count)), Arg.Any <CancellationToken>()); }
public async Task GivenAnOrchestratorTaskAndWrongEtag_WhenOrchestratorTaskStart_ThenTaskShouldFailedWithDetails() { IImportOrchestratorTaskDataStoreOperation fhirDataBulkImportOperation = Substitute.For <IImportOrchestratorTaskDataStoreOperation>(); IContextUpdater contextUpdater = Substitute.For <IContextUpdater>(); RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >(); ILoggerFactory loggerFactory = new NullLoggerFactory(); IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>(); ISequenceIdGenerator <long> sequenceIdGenerator = Substitute.For <ISequenceIdGenerator <long> >(); ImportOrchestratorTaskInputData importOrchestratorTaskInputData = new ImportOrchestratorTaskInputData(); ImportOrchestratorTaskContext importOrchestratorTaskContext = new ImportOrchestratorTaskContext(); ITaskManager taskManager = Substitute.For <ITaskManager>(); IMediator mediator = Substitute.For <IMediator>(); importOrchestratorTaskInputData.TaskId = Guid.NewGuid().ToString("N"); importOrchestratorTaskInputData.TaskCreateTime = Clock.UtcNow; importOrchestratorTaskInputData.BaseUri = new Uri("http://dummy"); var inputs = new List <InputResource>(); inputs.Add(new InputResource() { Type = "Resource", Url = new Uri("http://dummy"), Etag = "dummy" }); importOrchestratorTaskInputData.Input = inputs; importOrchestratorTaskInputData.InputFormat = "ndjson"; importOrchestratorTaskInputData.InputSource = new Uri("http://dummy"); importOrchestratorTaskInputData.MaxConcurrentProcessingTaskCount = 1; importOrchestratorTaskInputData.ProcessingTaskQueueId = "default"; importOrchestratorTaskInputData.RequestUri = new Uri("http://dummy"); integrationDataStoreClient.GetPropertiesAsync(Arg.Any <Uri>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { Dictionary <string, object> properties = new Dictionary <string, object>(); properties[IntegrationDataStoreClientConstants.BlobPropertyETag] = "test"; properties[IntegrationDataStoreClientConstants.BlobPropertyLength] = 1000L; return(properties); }); sequenceIdGenerator.GetCurrentSequenceId().Returns(_ => 0L); ImportOrchestratorTask orchestratorTask = new ImportOrchestratorTask( mediator, importOrchestratorTaskInputData, importOrchestratorTaskContext, taskManager, sequenceIdGenerator, contextUpdater, contextAccessor, fhirDataBulkImportOperation, integrationDataStoreClient, loggerFactory); TaskResultData result = await orchestratorTask.ExecuteAsync(); ImportTaskErrorResult resultDetails = JsonConvert.DeserializeObject <ImportTaskErrorResult>(result.ResultData); Assert.Equal(TaskResult.Fail, result.Result); Assert.Equal(HttpStatusCode.BadRequest, resultDetails.HttpStatusCode); Assert.NotEmpty(resultDetails.ErrorMessage); _ = mediator.Received().Publish( Arg.Is <ImportTaskMetricsNotification>( notification => notification.Id == importOrchestratorTaskInputData.TaskId && notification.Status == TaskResult.Fail.ToString() && notification.CreatedTime == importOrchestratorTaskInputData.TaskCreateTime && notification.DataSize == null && notification.SucceedCount == null && notification.FailedCount == null), Arg.Any <CancellationToken>()); }
public async Task GivenAnOrchestratorTask_WhenCancelBefore_ThenCanceledResultShouldBeReturn() { IImportOrchestratorTaskDataStoreOperation fhirDataBulkImportOperation = Substitute.For <IImportOrchestratorTaskDataStoreOperation>(); IContextUpdater contextUpdater = Substitute.For <IContextUpdater>(); RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >(); ILoggerFactory loggerFactory = new NullLoggerFactory(); IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>(); ISequenceIdGenerator <long> sequenceIdGenerator = Substitute.For <ISequenceIdGenerator <long> >(); IMediator mediator = Substitute.For <IMediator>(); ImportOrchestratorTaskInputData importOrchestratorTaskInputData = new ImportOrchestratorTaskInputData(); ImportOrchestratorTaskContext importOrchestratorTaskContext = new ImportOrchestratorTaskContext(); List <(long begin, long end)> surrogatedIdRanges = new List <(long begin, long end)>(); TestTaskManager taskManager = new TestTaskManager(t => { if (t == null) { return(null); } ImportProcessingTaskInputData processingInput = JsonConvert.DeserializeObject <ImportProcessingTaskInputData>(t.InputData); ImportProcessingTaskResult processingResult = new ImportProcessingTaskResult(); processingResult.ResourceType = processingInput.ResourceType; processingResult.SucceedCount = 1; processingResult.FailedCount = 1; processingResult.ErrorLogLocation = "http://dummy/error"; surrogatedIdRanges.Add((processingInput.BeginSequenceId, processingInput.EndSequenceId)); t.Result = JsonConvert.SerializeObject(new TaskResultData(TaskResult.Success, JsonConvert.SerializeObject(processingResult))); t.Status = TaskManagement.TaskStatus.Completed; return(t); }); importOrchestratorTaskInputData.TaskId = Guid.NewGuid().ToString("N"); importOrchestratorTaskInputData.TaskCreateTime = Clock.UtcNow; importOrchestratorTaskInputData.BaseUri = new Uri("http://dummy"); var inputs = new List <InputResource>(); inputs.Add(new InputResource() { Type = "Resource", Url = new Uri($"http://dummy") }); importOrchestratorTaskInputData.Input = inputs; importOrchestratorTaskInputData.InputFormat = "ndjson"; importOrchestratorTaskInputData.InputSource = new Uri("http://dummy"); importOrchestratorTaskInputData.MaxConcurrentProcessingTaskCount = 1; importOrchestratorTaskInputData.ProcessingTaskQueueId = "default"; importOrchestratorTaskInputData.RequestUri = new Uri("http://dummy"); integrationDataStoreClient.GetPropertiesAsync(Arg.Any <Uri>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { Dictionary <string, object> properties = new Dictionary <string, object>(); properties[IntegrationDataStoreClientConstants.BlobPropertyETag] = "test"; properties[IntegrationDataStoreClientConstants.BlobPropertyLength] = 1000L; return(properties); }); string latestContext = null; contextUpdater.UpdateContextAsync(Arg.Any <string>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { latestContext = (string)callInfo[0]; return(Task.CompletedTask); }); sequenceIdGenerator.GetCurrentSequenceId().Returns <long>(_ => 0L); ImportOrchestratorTask orchestratorTask = new ImportOrchestratorTask( mediator, importOrchestratorTaskInputData, importOrchestratorTaskContext, taskManager, sequenceIdGenerator, contextUpdater, contextAccessor, fhirDataBulkImportOperation, integrationDataStoreClient, loggerFactory); orchestratorTask.PollingFrequencyInSeconds = 0; orchestratorTask.Cancel(); TaskResultData taskResult = await orchestratorTask.ExecuteAsync(); Assert.Equal(TaskResult.Canceled, taskResult.Result); _ = mediator.Received().Publish( Arg.Is <ImportTaskMetricsNotification>( notification => notification.Id == importOrchestratorTaskInputData.TaskId && notification.Status == TaskResult.Canceled.ToString() && notification.CreatedTime == importOrchestratorTaskInputData.TaskCreateTime && notification.SucceedCount == null && notification.FailedCount == null), Arg.Any <CancellationToken>()); }
public async Task GivenAnOrchestratorTask_WhenSubTaskFailed_ThenImportProcessingExceptionShouldBeThrowAndContextUpdated() { IImportOrchestratorTaskDataStoreOperation fhirDataBulkImportOperation = Substitute.For <IImportOrchestratorTaskDataStoreOperation>(); IContextUpdater contextUpdater = Substitute.For <IContextUpdater>(); RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >(); ILoggerFactory loggerFactory = new NullLoggerFactory(); IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>(); ISequenceIdGenerator <long> sequenceIdGenerator = Substitute.For <ISequenceIdGenerator <long> >(); ImportOrchestratorTaskInputData importOrchestratorTaskInputData = new ImportOrchestratorTaskInputData(); ImportOrchestratorTaskContext importOrchestratorTaskContext = new ImportOrchestratorTaskContext(); List <(long begin, long end)> surrogatedIdRanges = new List <(long begin, long end)>(); TestTaskManager taskManager = new TestTaskManager(t => { if (t == null) { return(null); } TaskResultData resultData = new TaskResultData(); resultData.Result = TaskResult.Fail; resultData.ResultData = "error"; t.Result = JsonConvert.SerializeObject(resultData); t.Status = TaskManagement.TaskStatus.Completed; return(t); }); importOrchestratorTaskInputData.TaskId = Guid.NewGuid().ToString("N"); importOrchestratorTaskInputData.TaskCreateTime = Clock.UtcNow; importOrchestratorTaskInputData.BaseUri = new Uri("http://dummy"); var inputs = new List <InputResource>(); inputs.Add(new InputResource() { Type = "Resource", Url = new Uri($"http://dummy") }); importOrchestratorTaskInputData.Input = inputs; importOrchestratorTaskInputData.InputFormat = "ndjson"; importOrchestratorTaskInputData.InputSource = new Uri("http://dummy"); importOrchestratorTaskInputData.MaxConcurrentProcessingTaskCount = 1; importOrchestratorTaskInputData.ProcessingTaskQueueId = "default"; importOrchestratorTaskInputData.RequestUri = new Uri("http://dummy"); integrationDataStoreClient.GetPropertiesAsync(Arg.Any <Uri>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { Dictionary <string, object> properties = new Dictionary <string, object>(); properties[IntegrationDataStoreClientConstants.BlobPropertyETag] = "test"; properties[IntegrationDataStoreClientConstants.BlobPropertyLength] = 1000L; return(properties); }); string latestContext = null; contextUpdater.UpdateContextAsync(Arg.Any <string>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { latestContext = (string)callInfo[0]; return(Task.CompletedTask); }); sequenceIdGenerator.GetCurrentSequenceId().Returns <long>(_ => 0L); ImportOrchestratorTask orchestratorTask = new ImportOrchestratorTask( importOrchestratorTaskInputData, importOrchestratorTaskContext, taskManager, sequenceIdGenerator, contextUpdater, contextAccessor, fhirDataBulkImportOperation, integrationDataStoreClient, loggerFactory); orchestratorTask.PollingFrequencyInSeconds = 0; TaskResultData taskResultData = await orchestratorTask.ExecuteAsync(); Assert.Equal(TaskResult.Fail, taskResultData.Result); ImportOrchestratorTaskContext context = JsonConvert.DeserializeObject <ImportOrchestratorTaskContext>(latestContext); Assert.Equal(ImportOrchestratorTaskProgress.SubTaskRecordsGenerated, context.Progress); }