public async Task GivenImportTaskInput_WhenExceptionThrowForImport_ThenContextShouldBeUpdatedBeforeFailure() { long currentIndex = 100; ImportProcessingTaskInputData inputData = GetInputData(); ImportProcessingProgress progress = new ImportProcessingProgress(); IImportResourceLoader loader = Substitute.For <IImportResourceLoader>(); IResourceBulkImporter importer = Substitute.For <IResourceBulkImporter>(); IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>(); IImportErrorStoreFactory importErrorStoreFactory = Substitute.For <IImportErrorStoreFactory>(); IContextUpdater contextUpdater = Substitute.For <IContextUpdater>(); RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >(); ILoggerFactory loggerFactory = new NullLoggerFactory(); loader.LoadResources(Arg.Any <string>(), Arg.Any <long>(), Arg.Any <string>(), Arg.Any <Func <long, long> >(), Arg.Any <CancellationToken>()) .Returns(callInfo => { long startIndex = (long)callInfo[1]; Func <long, long> idGenerator = (Func <long, long>)callInfo[3]; Channel <ImportResource> resourceChannel = Channel.CreateUnbounded <ImportResource>(); resourceChannel.Writer.Complete(); return(resourceChannel, Task.CompletedTask); }); importer.Import(Arg.Any <Channel <ImportResource> >(), Arg.Any <IImportErrorStore>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { Channel <ImportProcessingProgress> progressChannel = Channel.CreateUnbounded <ImportProcessingProgress>(); Task loadTask = Task.Run(async() => { try { ImportProcessingProgress progress = new ImportProcessingProgress(); progress.CurrentIndex = currentIndex; await progressChannel.Writer.WriteAsync(progress); throw new InvalidOperationException(); } finally { progressChannel.Writer.Complete(); } }); return(progressChannel, loadTask); }); string context = null; contextUpdater.UpdateContextAsync(Arg.Any <string>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { context = (string)callInfo[0]; return(Task.CompletedTask); }); ImportProcessingTask task = new ImportProcessingTask( inputData, progress, loader, importer, importErrorStoreFactory, contextUpdater, contextAccessor, loggerFactory); await Assert.ThrowsAsync <RetriableTaskException>(() => task.ExecuteAsync()); ImportProcessingProgress progressForContext = JsonConvert.DeserializeObject <ImportProcessingProgress>(context); Assert.Equal(progressForContext.CurrentIndex, currentIndex); }
private static async Task VerifyCommonImportTaskAsync(ImportProcessingTaskInputData inputData, ImportProcessingProgress progress) { long startIndexFromProgress = progress.CurrentIndex; long succeedCountFromProgress = progress.SucceedImportCount; long failedCountFromProgress = progress.FailedImportCount; IImportResourceLoader loader = Substitute.For <IImportResourceLoader>(); IResourceBulkImporter importer = Substitute.For <IResourceBulkImporter>(); IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>(); IImportErrorStoreFactory importErrorStoreFactory = Substitute.For <IImportErrorStoreFactory>(); IContextUpdater contextUpdater = Substitute.For <IContextUpdater>(); RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >(); ILoggerFactory loggerFactory = new NullLoggerFactory(); long cleanStart = -1; long cleanEnd = -1; importer.CleanResourceAsync(Arg.Any <ImportProcessingTaskInputData>(), Arg.Any <ImportProcessingProgress>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { var inputData = (ImportProcessingTaskInputData)callInfo[0]; var progress = (ImportProcessingProgress)callInfo[1]; long beginSequenceId = inputData.BeginSequenceId; long endSequenceId = inputData.EndSequenceId; long endIndex = progress.CurrentIndex; cleanStart = beginSequenceId + endIndex; cleanEnd = endSequenceId; return(Task.CompletedTask); }); loader.LoadResources(Arg.Any <string>(), Arg.Any <long>(), Arg.Any <string>(), Arg.Any <Func <long, long> >(), Arg.Any <CancellationToken>()) .Returns(callInfo => { long startIndex = (long)callInfo[1]; Func <long, long> idGenerator = (Func <long, long>)callInfo[3]; Channel <ImportResource> resourceChannel = Channel.CreateUnbounded <ImportResource>(); Task loadTask = Task.Run(async() => { ResourceWrapper resourceWrapper = new ResourceWrapper( Guid.NewGuid().ToString(), "0", "Dummy", new RawResource(Guid.NewGuid().ToString(), Fhir.Core.Models.FhirResourceFormat.Json, true), new ResourceRequest("POST"), DateTimeOffset.UtcNow, false, null, null, null, "SearchParam"); await resourceChannel.Writer.WriteAsync(new ImportResource(idGenerator(startIndex), startIndex, resourceWrapper)); await resourceChannel.Writer.WriteAsync(new ImportResource(idGenerator(startIndex + 1), startIndex + 1, "Error")); resourceChannel.Writer.Complete(); }); return(resourceChannel, loadTask); }); importer.Import(Arg.Any <Channel <ImportResource> >(), Arg.Any <IImportErrorStore>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { Channel <ImportResource> resourceChannel = (Channel <ImportResource>)callInfo[0]; Channel <ImportProcessingProgress> progressChannel = Channel.CreateUnbounded <ImportProcessingProgress>(); Task loadTask = Task.Run(async() => { ImportProcessingProgress progress = new ImportProcessingProgress(); await foreach (ImportResource resource in resourceChannel.Reader.ReadAllAsync()) { if (string.IsNullOrEmpty(resource.ImportError)) { progress.SucceedImportCount++; } else { progress.FailedImportCount++; } progress.CurrentIndex = resource.Index + 1; } await progressChannel.Writer.WriteAsync(progress); progressChannel.Writer.Complete(); }); return(progressChannel, loadTask); }); string context = null; contextUpdater.UpdateContextAsync(Arg.Any <string>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { context = (string)callInfo[0]; return(Task.CompletedTask); }); progress.NeedCleanData = true; ImportProcessingTask task = new ImportProcessingTask( inputData, progress, loader, importer, importErrorStoreFactory, contextUpdater, contextAccessor, loggerFactory); TaskResultData taskResult = await task.ExecuteAsync(); Assert.Equal(TaskResult.Success, taskResult.Result); ImportProcessingTaskResult result = JsonConvert.DeserializeObject <ImportProcessingTaskResult>(taskResult.ResultData); Assert.Equal(1 + failedCountFromProgress, result.FailedCount); Assert.Equal(1 + succeedCountFromProgress, result.SucceedCount); ImportProcessingProgress progressForContext = JsonConvert.DeserializeObject <ImportProcessingProgress>(context); Assert.Equal(progressForContext.SucceedImportCount, result.SucceedCount); Assert.Equal(progressForContext.FailedImportCount, result.FailedCount); Assert.Equal(startIndexFromProgress + 2, progressForContext.CurrentIndex); Assert.Equal(startIndexFromProgress, cleanStart); Assert.Equal(inputData.EndSequenceId, cleanEnd); }
public async Task GivenAnOrchestratorTask_WhenCancelBefore_ThenCanceledResultShouldBeReturn() { IImportOrchestratorTaskDataStoreOperation fhirDataBulkImportOperation = Substitute.For <IImportOrchestratorTaskDataStoreOperation>(); IContextUpdater contextUpdater = Substitute.For <IContextUpdater>(); RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >(); ILoggerFactory loggerFactory = new NullLoggerFactory(); IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>(); ISequenceIdGenerator <long> sequenceIdGenerator = Substitute.For <ISequenceIdGenerator <long> >(); IMediator mediator = Substitute.For <IMediator>(); ImportOrchestratorTaskInputData importOrchestratorTaskInputData = new ImportOrchestratorTaskInputData(); ImportOrchestratorTaskContext importOrchestratorTaskContext = new ImportOrchestratorTaskContext(); List <(long begin, long end)> surrogatedIdRanges = new List <(long begin, long end)>(); TestTaskManager taskManager = new TestTaskManager(t => { if (t == null) { return(null); } ImportProcessingTaskInputData processingInput = JsonConvert.DeserializeObject <ImportProcessingTaskInputData>(t.InputData); ImportProcessingTaskResult processingResult = new ImportProcessingTaskResult(); processingResult.ResourceType = processingInput.ResourceType; processingResult.SucceedCount = 1; processingResult.FailedCount = 1; processingResult.ErrorLogLocation = "http://dummy/error"; surrogatedIdRanges.Add((processingInput.BeginSequenceId, processingInput.EndSequenceId)); t.Result = JsonConvert.SerializeObject(new TaskResultData(TaskResult.Success, JsonConvert.SerializeObject(processingResult))); t.Status = TaskManagement.TaskStatus.Completed; return(t); }); importOrchestratorTaskInputData.TaskId = Guid.NewGuid().ToString("N"); importOrchestratorTaskInputData.TaskCreateTime = Clock.UtcNow; importOrchestratorTaskInputData.BaseUri = new Uri("http://dummy"); var inputs = new List <InputResource>(); inputs.Add(new InputResource() { Type = "Resource", Url = new Uri($"http://dummy") }); importOrchestratorTaskInputData.Input = inputs; importOrchestratorTaskInputData.InputFormat = "ndjson"; importOrchestratorTaskInputData.InputSource = new Uri("http://dummy"); importOrchestratorTaskInputData.MaxConcurrentProcessingTaskCount = 1; importOrchestratorTaskInputData.ProcessingTaskQueueId = "default"; importOrchestratorTaskInputData.RequestUri = new Uri("http://dummy"); integrationDataStoreClient.GetPropertiesAsync(Arg.Any <Uri>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { Dictionary <string, object> properties = new Dictionary <string, object>(); properties[IntegrationDataStoreClientConstants.BlobPropertyETag] = "test"; properties[IntegrationDataStoreClientConstants.BlobPropertyLength] = 1000L; return(properties); }); string latestContext = null; contextUpdater.UpdateContextAsync(Arg.Any <string>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { latestContext = (string)callInfo[0]; return(Task.CompletedTask); }); sequenceIdGenerator.GetCurrentSequenceId().Returns <long>(_ => 0L); ImportOrchestratorTask orchestratorTask = new ImportOrchestratorTask( mediator, importOrchestratorTaskInputData, importOrchestratorTaskContext, taskManager, sequenceIdGenerator, contextUpdater, contextAccessor, fhirDataBulkImportOperation, integrationDataStoreClient, loggerFactory); orchestratorTask.PollingFrequencyInSeconds = 0; orchestratorTask.Cancel(); TaskResultData taskResult = await orchestratorTask.ExecuteAsync(); Assert.Equal(TaskResult.Canceled, taskResult.Result); _ = mediator.Received().Publish( Arg.Is <ImportTaskMetricsNotification>( notification => notification.Id == importOrchestratorTaskInputData.TaskId && notification.Status == TaskResult.Canceled.ToString() && notification.CreatedTime == importOrchestratorTaskInputData.TaskCreateTime && notification.SucceedCount == null && notification.FailedCount == null), Arg.Any <CancellationToken>()); }
public async Task <TaskResultData> ExecuteAsync() { var fhirRequestContext = new FhirRequestContext( method: "Import", uriString: _inputData.UriString, baseUriString: _inputData.BaseUriString, correlationId: _inputData.TaskId, requestHeaders: new Dictionary <string, StringValues>(), responseHeaders: new Dictionary <string, StringValues>()) { IsBackgroundTask = true, }; _contextAccessor.RequestContext = fhirRequestContext; CancellationToken cancellationToken = _cancellationTokenSource.Token; long succeedImportCount = _importProgress.SucceedImportCount; long failedImportCount = _importProgress.FailedImportCount; ImportProcessingTaskResult result = new ImportProcessingTaskResult(); result.ResourceType = _inputData.ResourceType; try { if (cancellationToken.IsCancellationRequested) { throw new OperationCanceledException(); } Func <long, long> sequenceIdGenerator = (index) => _inputData.BeginSequenceId + index; // Clean resources before import start await _resourceBulkImporter.CleanResourceAsync(_inputData, _importProgress, cancellationToken); _importProgress.NeedCleanData = true; await _contextUpdater.UpdateContextAsync(JsonConvert.SerializeObject(_importProgress), cancellationToken); // Initialize error store IImportErrorStore importErrorStore = await _importErrorStoreFactory.InitializeAsync(GetErrorFileName(), cancellationToken); result.ErrorLogLocation = importErrorStore.ErrorFileLocation; // Load and parse resource from bulk resource (Channel <ImportResource> importResourceChannel, Task loadTask) = _importResourceLoader.LoadResources(_inputData.ResourceLocation, _importProgress.CurrentIndex, _inputData.ResourceType, sequenceIdGenerator, cancellationToken); // Import to data store (Channel <ImportProcessingProgress> progressChannel, Task importTask) = _resourceBulkImporter.Import(importResourceChannel, importErrorStore, cancellationToken); // Update progress for checkpoints await foreach (ImportProcessingProgress progress in progressChannel.Reader.ReadAllAsync()) { if (cancellationToken.IsCancellationRequested) { throw new OperationCanceledException("Import task is canceled by user."); } _importProgress.SucceedImportCount = progress.SucceedImportCount + succeedImportCount; _importProgress.FailedImportCount = progress.FailedImportCount + failedImportCount; _importProgress.CurrentIndex = progress.CurrentIndex; result.SucceedCount = _importProgress.SucceedImportCount; result.FailedCount = _importProgress.FailedImportCount; _logger.LogInformation("Import task progress: {0}", JsonConvert.SerializeObject(_importProgress)); try { await _contextUpdater.UpdateContextAsync(JsonConvert.SerializeObject(_importProgress), cancellationToken); } catch (Exception ex) { // ignore exception for progresss update _logger.LogInformation(ex, "Failed to update context."); } } // Pop up exception during load & import // Put import task before load task for resource channel full and blocking issue. try { await importTask; } catch (TaskCanceledException) { throw; } catch (OperationCanceledException) { throw; } catch (Exception ex) { _logger.LogError(ex, "Failed to import data."); throw new RetriableTaskException("Failed to import data.", ex); } try { await loadTask; } catch (TaskCanceledException) { throw; } catch (OperationCanceledException) { throw; } catch (Exception ex) { _logger.LogError(ex, "Failed to load data."); throw new RetriableTaskException("Failed to load data", ex); } return(new TaskResultData(TaskResult.Success, JsonConvert.SerializeObject(result))); } catch (TaskCanceledException canceledEx) { _logger.LogInformation(canceledEx, "Data processing task is canceled."); await CleanResourceForFailureAsync(canceledEx); return(new TaskResultData(TaskResult.Canceled, JsonConvert.SerializeObject(result))); } catch (OperationCanceledException canceledEx) { _logger.LogInformation(canceledEx, "Data processing task is canceled."); await CleanResourceForFailureAsync(canceledEx); return(new TaskResultData(TaskResult.Canceled, JsonConvert.SerializeObject(result))); } catch (RetriableTaskException retriableEx) { _logger.LogInformation(retriableEx, "Error in data processing task."); await CleanResourceForFailureAsync(retriableEx); throw; } catch (Exception ex) { _logger.LogInformation(ex, "Critical error in data processing task."); await CleanResourceForFailureAsync(ex); throw new RetriableTaskException(ex.Message); } finally { if (!_cancellationTokenSource.IsCancellationRequested) { _cancellationTokenSource.Cancel(); } } }
public async Task GivenAnOrchestratorTask_WhenSubTaskFailed_ThenImportProcessingExceptionShouldBeThrowAndContextUpdated() { IImportOrchestratorTaskDataStoreOperation fhirDataBulkImportOperation = Substitute.For <IImportOrchestratorTaskDataStoreOperation>(); IContextUpdater contextUpdater = Substitute.For <IContextUpdater>(); RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >(); ILoggerFactory loggerFactory = new NullLoggerFactory(); IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>(); ISequenceIdGenerator <long> sequenceIdGenerator = Substitute.For <ISequenceIdGenerator <long> >(); ImportOrchestratorTaskInputData importOrchestratorTaskInputData = new ImportOrchestratorTaskInputData(); ImportOrchestratorTaskContext importOrchestratorTaskContext = new ImportOrchestratorTaskContext(); List <(long begin, long end)> surrogatedIdRanges = new List <(long begin, long end)>(); TestTaskManager taskManager = new TestTaskManager(t => { if (t == null) { return(null); } TaskResultData resultData = new TaskResultData(); resultData.Result = TaskResult.Fail; resultData.ResultData = "error"; t.Result = JsonConvert.SerializeObject(resultData); t.Status = TaskManagement.TaskStatus.Completed; return(t); }); importOrchestratorTaskInputData.TaskId = Guid.NewGuid().ToString("N"); importOrchestratorTaskInputData.TaskCreateTime = Clock.UtcNow; importOrchestratorTaskInputData.BaseUri = new Uri("http://dummy"); var inputs = new List <InputResource>(); inputs.Add(new InputResource() { Type = "Resource", Url = new Uri($"http://dummy") }); importOrchestratorTaskInputData.Input = inputs; importOrchestratorTaskInputData.InputFormat = "ndjson"; importOrchestratorTaskInputData.InputSource = new Uri("http://dummy"); importOrchestratorTaskInputData.MaxConcurrentProcessingTaskCount = 1; importOrchestratorTaskInputData.ProcessingTaskQueueId = "default"; importOrchestratorTaskInputData.RequestUri = new Uri("http://dummy"); integrationDataStoreClient.GetPropertiesAsync(Arg.Any <Uri>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { Dictionary <string, object> properties = new Dictionary <string, object>(); properties[IntegrationDataStoreClientConstants.BlobPropertyETag] = "test"; properties[IntegrationDataStoreClientConstants.BlobPropertyLength] = 1000L; return(properties); }); string latestContext = null; contextUpdater.UpdateContextAsync(Arg.Any <string>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { latestContext = (string)callInfo[0]; return(Task.CompletedTask); }); sequenceIdGenerator.GetCurrentSequenceId().Returns <long>(_ => 0L); ImportOrchestratorTask orchestratorTask = new ImportOrchestratorTask( importOrchestratorTaskInputData, importOrchestratorTaskContext, taskManager, sequenceIdGenerator, contextUpdater, contextAccessor, fhirDataBulkImportOperation, integrationDataStoreClient, loggerFactory); orchestratorTask.PollingFrequencyInSeconds = 0; TaskResultData taskResultData = await orchestratorTask.ExecuteAsync(); Assert.Equal(TaskResult.Fail, taskResultData.Result); ImportOrchestratorTaskContext context = JsonConvert.DeserializeObject <ImportOrchestratorTaskContext>(latestContext); Assert.Equal(ImportOrchestratorTaskProgress.SubTaskRecordsGenerated, context.Progress); }