private static async Task VerifyCommonOrchestratorTaskAsync(int inputFileCount, int concurrentCount, int resumeFrom = -1) { IImportOrchestratorTaskDataStoreOperation fhirDataBulkImportOperation = Substitute.For <IImportOrchestratorTaskDataStoreOperation>(); IContextUpdater contextUpdater = Substitute.For <IContextUpdater>(); RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >(); ILoggerFactory loggerFactory = new NullLoggerFactory(); IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>(); ISequenceIdGenerator <long> sequenceIdGenerator = Substitute.For <ISequenceIdGenerator <long> >(); IMediator mediator = Substitute.For <IMediator>(); ImportOrchestratorTaskInputData importOrchestratorTaskInputData = new ImportOrchestratorTaskInputData(); ImportOrchestratorTaskContext importOrchestratorTaskContext = new ImportOrchestratorTaskContext(); List <(long begin, long end)> surrogatedIdRanges = new List <(long begin, long end)>(); TestTaskManager taskManager = new TestTaskManager(t => { if (t == null) { return(null); } if (t.Status == TaskManagement.TaskStatus.Completed) { return(t); } ImportProcessingTaskInputData processingInput = JsonConvert.DeserializeObject <ImportProcessingTaskInputData>(t.InputData); ImportProcessingTaskResult processingResult = new ImportProcessingTaskResult(); processingResult.ResourceType = processingInput.ResourceType; processingResult.SucceedCount = 1; processingResult.FailedCount = 1; processingResult.ErrorLogLocation = "http://dummy/error"; surrogatedIdRanges.Add((processingInput.BeginSequenceId, processingInput.EndSequenceId)); t.Result = JsonConvert.SerializeObject(new TaskResultData(TaskResult.Success, JsonConvert.SerializeObject(processingResult))); t.Status = TaskManagement.TaskStatus.Completed; return(t); }); importOrchestratorTaskInputData.TaskId = Guid.NewGuid().ToString("N"); importOrchestratorTaskInputData.TaskCreateTime = Clock.UtcNow; importOrchestratorTaskInputData.BaseUri = new Uri("http://dummy"); var inputs = new List <InputResource>(); bool resumeMode = resumeFrom >= 0; for (int i = 0; i < inputFileCount; ++i) { string location = $"http://dummy/{i}"; inputs.Add(new InputResource() { Type = "Resource", Url = new Uri(location) }); if (resumeMode) { if (i <= resumeFrom) { TaskInfo taskInfo = new TaskInfo(); taskInfo.TaskId = Guid.NewGuid().ToString("N"); ImportProcessingTaskResult processingResult = new ImportProcessingTaskResult(); processingResult.ResourceType = "Resource"; processingResult.SucceedCount = 1; processingResult.FailedCount = 1; processingResult.ErrorLogLocation = "http://dummy/error"; taskInfo.Result = JsonConvert.SerializeObject(new TaskResultData(TaskResult.Success, JsonConvert.SerializeObject(processingResult))); taskInfo.Status = TaskManagement.TaskStatus.Completed; await taskManager.CreateTaskAsync(taskInfo, false, CancellationToken.None); importOrchestratorTaskContext.DataProcessingTasks[new Uri(location)] = taskInfo; } else { TaskInfo taskInfo = new TaskInfo(); taskInfo.TaskId = Guid.NewGuid().ToString("N"); ImportProcessingTaskInputData processingInput = new ImportProcessingTaskInputData(); processingInput.BaseUriString = "http://dummy"; processingInput.BeginSequenceId = i; processingInput.EndSequenceId = i + 1; processingInput.ResourceType = "Resource"; taskInfo.InputData = JsonConvert.SerializeObject(processingInput); await taskManager.CreateTaskAsync(taskInfo, false, CancellationToken.None); importOrchestratorTaskContext.DataProcessingTasks[new Uri(location)] = taskInfo; } importOrchestratorTaskContext.Progress = ImportOrchestratorTaskProgress.SubTaskRecordsGenerated; } } importOrchestratorTaskInputData.Input = inputs; importOrchestratorTaskInputData.InputFormat = "ndjson"; importOrchestratorTaskInputData.InputSource = new Uri("http://dummy"); importOrchestratorTaskInputData.MaxConcurrentProcessingTaskCount = concurrentCount; importOrchestratorTaskInputData.ProcessingTaskQueueId = "default"; importOrchestratorTaskInputData.RequestUri = new Uri("http://dummy"); integrationDataStoreClient.GetPropertiesAsync(Arg.Any <Uri>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { Dictionary <string, object> properties = new Dictionary <string, object>(); properties[IntegrationDataStoreClientConstants.BlobPropertyETag] = "test"; properties[IntegrationDataStoreClientConstants.BlobPropertyLength] = 1000L; return(properties); }); sequenceIdGenerator.GetCurrentSequenceId().Returns(_ => 0L); ImportOrchestratorTask orchestratorTask = new ImportOrchestratorTask( mediator, importOrchestratorTaskInputData, importOrchestratorTaskContext, taskManager, sequenceIdGenerator, contextUpdater, contextAccessor, fhirDataBulkImportOperation, integrationDataStoreClient, loggerFactory); orchestratorTask.PollingFrequencyInSeconds = 0; TaskResultData result = await orchestratorTask.ExecuteAsync(); ImportTaskResult resultDetails = JsonConvert.DeserializeObject <ImportTaskResult>(result.ResultData); Assert.Equal(TaskResult.Success, result.Result); Assert.Equal(inputFileCount, resultDetails.Output.Count); foreach (ImportOperationOutcome outcome in resultDetails.Output) { Assert.Equal(1, outcome.Count); Assert.NotNull(outcome.InputUrl); Assert.NotEmpty(outcome.Type); } Assert.Equal(inputFileCount, resultDetails.Error.Count); foreach (ImportFailedOperationOutcome outcome in resultDetails.Error) { Assert.Equal(1, outcome.Count); Assert.NotNull(outcome.InputUrl); Assert.NotEmpty(outcome.Type); Assert.NotEmpty(outcome.Url); } Assert.NotEmpty(resultDetails.Request); Assert.Equal(importOrchestratorTaskInputData.TaskCreateTime, resultDetails.TransactionTime); var orderedSurrogatedIdRanges = surrogatedIdRanges.OrderBy(r => r.begin).ToArray(); Assert.Equal(inputFileCount, orderedSurrogatedIdRanges.Length + resumeFrom + 1); for (int i = 0; i < orderedSurrogatedIdRanges.Length - 1; ++i) { Assert.True(orderedSurrogatedIdRanges[i].end > orderedSurrogatedIdRanges[i].begin); Assert.True(orderedSurrogatedIdRanges[i].end <= orderedSurrogatedIdRanges[i + 1].begin); } _ = mediator.Received().Publish( Arg.Is <ImportTaskMetricsNotification>( notification => notification.Id == importOrchestratorTaskInputData.TaskId && notification.Status == TaskResult.Success.ToString() && notification.CreatedTime == importOrchestratorTaskInputData.TaskCreateTime && notification.SucceedCount == resultDetails.Output.Sum(o => o.Count) && notification.FailedCount == resultDetails.Error.Sum(e => e.Count)), Arg.Any <CancellationToken>()); }
private static async Task VerifyCommonImportTaskAsync(ImportProcessingTaskInputData inputData, ImportProcessingProgress progress) { long startIndexFromProgress = progress.CurrentIndex; long succeedCountFromProgress = progress.SucceedImportCount; long failedCountFromProgress = progress.FailedImportCount; IImportResourceLoader loader = Substitute.For <IImportResourceLoader>(); IResourceBulkImporter importer = Substitute.For <IResourceBulkImporter>(); IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>(); IImportErrorStoreFactory importErrorStoreFactory = Substitute.For <IImportErrorStoreFactory>(); IContextUpdater contextUpdater = Substitute.For <IContextUpdater>(); RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >(); ILoggerFactory loggerFactory = new NullLoggerFactory(); long cleanStart = -1; long cleanEnd = -1; importer.CleanResourceAsync(Arg.Any <ImportProcessingTaskInputData>(), Arg.Any <ImportProcessingProgress>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { var inputData = (ImportProcessingTaskInputData)callInfo[0]; var progress = (ImportProcessingProgress)callInfo[1]; long beginSequenceId = inputData.BeginSequenceId; long endSequenceId = inputData.EndSequenceId; long endIndex = progress.CurrentIndex; cleanStart = beginSequenceId + endIndex; cleanEnd = endSequenceId; return(Task.CompletedTask); }); loader.LoadResources(Arg.Any <string>(), Arg.Any <long>(), Arg.Any <string>(), Arg.Any <Func <long, long> >(), Arg.Any <CancellationToken>()) .Returns(callInfo => { long startIndex = (long)callInfo[1]; Func <long, long> idGenerator = (Func <long, long>)callInfo[3]; Channel <ImportResource> resourceChannel = Channel.CreateUnbounded <ImportResource>(); Task loadTask = Task.Run(async() => { ResourceWrapper resourceWrapper = new ResourceWrapper( Guid.NewGuid().ToString(), "0", "Dummy", new RawResource(Guid.NewGuid().ToString(), Fhir.Core.Models.FhirResourceFormat.Json, true), new ResourceRequest("POST"), DateTimeOffset.UtcNow, false, null, null, null, "SearchParam"); await resourceChannel.Writer.WriteAsync(new ImportResource(idGenerator(startIndex), startIndex, resourceWrapper)); await resourceChannel.Writer.WriteAsync(new ImportResource(idGenerator(startIndex + 1), startIndex + 1, "Error")); resourceChannel.Writer.Complete(); }); return(resourceChannel, loadTask); }); importer.Import(Arg.Any <Channel <ImportResource> >(), Arg.Any <IImportErrorStore>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { Channel <ImportResource> resourceChannel = (Channel <ImportResource>)callInfo[0]; Channel <ImportProcessingProgress> progressChannel = Channel.CreateUnbounded <ImportProcessingProgress>(); Task loadTask = Task.Run(async() => { ImportProcessingProgress progress = new ImportProcessingProgress(); await foreach (ImportResource resource in resourceChannel.Reader.ReadAllAsync()) { if (string.IsNullOrEmpty(resource.ImportError)) { progress.SucceedImportCount++; } else { progress.FailedImportCount++; } progress.CurrentIndex = resource.Index + 1; } await progressChannel.Writer.WriteAsync(progress); progressChannel.Writer.Complete(); }); return(progressChannel, loadTask); }); string context = null; contextUpdater.UpdateContextAsync(Arg.Any <string>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { context = (string)callInfo[0]; return(Task.CompletedTask); }); progress.NeedCleanData = true; ImportProcessingTask task = new ImportProcessingTask( inputData, progress, loader, importer, importErrorStoreFactory, contextUpdater, contextAccessor, loggerFactory); TaskResultData taskResult = await task.ExecuteAsync(); Assert.Equal(TaskResult.Success, taskResult.Result); ImportProcessingTaskResult result = JsonConvert.DeserializeObject <ImportProcessingTaskResult>(taskResult.ResultData); Assert.Equal(1 + failedCountFromProgress, result.FailedCount); Assert.Equal(1 + succeedCountFromProgress, result.SucceedCount); ImportProcessingProgress progressForContext = JsonConvert.DeserializeObject <ImportProcessingProgress>(context); Assert.Equal(progressForContext.SucceedImportCount, result.SucceedCount); Assert.Equal(progressForContext.FailedImportCount, result.FailedCount); Assert.Equal(startIndexFromProgress + 2, progressForContext.CurrentIndex); Assert.Equal(startIndexFromProgress, cleanStart); Assert.Equal(inputData.EndSequenceId, cleanEnd); }
public async Task GivenAnOrchestratorTask_WhenCancelBefore_ThenCanceledResultShouldBeReturn() { IImportOrchestratorTaskDataStoreOperation fhirDataBulkImportOperation = Substitute.For <IImportOrchestratorTaskDataStoreOperation>(); IContextUpdater contextUpdater = Substitute.For <IContextUpdater>(); RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >(); ILoggerFactory loggerFactory = new NullLoggerFactory(); IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>(); ISequenceIdGenerator <long> sequenceIdGenerator = Substitute.For <ISequenceIdGenerator <long> >(); IMediator mediator = Substitute.For <IMediator>(); ImportOrchestratorTaskInputData importOrchestratorTaskInputData = new ImportOrchestratorTaskInputData(); ImportOrchestratorTaskContext importOrchestratorTaskContext = new ImportOrchestratorTaskContext(); List <(long begin, long end)> surrogatedIdRanges = new List <(long begin, long end)>(); TestTaskManager taskManager = new TestTaskManager(t => { if (t == null) { return(null); } ImportProcessingTaskInputData processingInput = JsonConvert.DeserializeObject <ImportProcessingTaskInputData>(t.InputData); ImportProcessingTaskResult processingResult = new ImportProcessingTaskResult(); processingResult.ResourceType = processingInput.ResourceType; processingResult.SucceedCount = 1; processingResult.FailedCount = 1; processingResult.ErrorLogLocation = "http://dummy/error"; surrogatedIdRanges.Add((processingInput.BeginSequenceId, processingInput.EndSequenceId)); t.Result = JsonConvert.SerializeObject(new TaskResultData(TaskResult.Success, JsonConvert.SerializeObject(processingResult))); t.Status = TaskManagement.TaskStatus.Completed; return(t); }); importOrchestratorTaskInputData.TaskId = Guid.NewGuid().ToString("N"); importOrchestratorTaskInputData.TaskCreateTime = Clock.UtcNow; importOrchestratorTaskInputData.BaseUri = new Uri("http://dummy"); var inputs = new List <InputResource>(); inputs.Add(new InputResource() { Type = "Resource", Url = new Uri($"http://dummy") }); importOrchestratorTaskInputData.Input = inputs; importOrchestratorTaskInputData.InputFormat = "ndjson"; importOrchestratorTaskInputData.InputSource = new Uri("http://dummy"); importOrchestratorTaskInputData.MaxConcurrentProcessingTaskCount = 1; importOrchestratorTaskInputData.ProcessingTaskQueueId = "default"; importOrchestratorTaskInputData.RequestUri = new Uri("http://dummy"); integrationDataStoreClient.GetPropertiesAsync(Arg.Any <Uri>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { Dictionary <string, object> properties = new Dictionary <string, object>(); properties[IntegrationDataStoreClientConstants.BlobPropertyETag] = "test"; properties[IntegrationDataStoreClientConstants.BlobPropertyLength] = 1000L; return(properties); }); string latestContext = null; contextUpdater.UpdateContextAsync(Arg.Any <string>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { latestContext = (string)callInfo[0]; return(Task.CompletedTask); }); sequenceIdGenerator.GetCurrentSequenceId().Returns <long>(_ => 0L); ImportOrchestratorTask orchestratorTask = new ImportOrchestratorTask( mediator, importOrchestratorTaskInputData, importOrchestratorTaskContext, taskManager, sequenceIdGenerator, contextUpdater, contextAccessor, fhirDataBulkImportOperation, integrationDataStoreClient, loggerFactory); orchestratorTask.PollingFrequencyInSeconds = 0; orchestratorTask.Cancel(); TaskResultData taskResult = await orchestratorTask.ExecuteAsync(); Assert.Equal(TaskResult.Canceled, taskResult.Result); _ = mediator.Received().Publish( Arg.Is <ImportTaskMetricsNotification>( notification => notification.Id == importOrchestratorTaskInputData.TaskId && notification.Status == TaskResult.Canceled.ToString() && notification.CreatedTime == importOrchestratorTaskInputData.TaskCreateTime && notification.SucceedCount == null && notification.FailedCount == null), Arg.Any <CancellationToken>()); }