public TaskFactory( IImportResourceLoader importResourceLoader, IResourceBulkImporter resourceBulkImporter, IImportErrorStoreFactory importErrorStoreFactory, IImportOrchestratorTaskDataStoreOperation importOrchestratorTaskDataStoreOperation, IContextUpdaterFactory contextUpdaterFactory, ITaskManager taskmanager, ISequenceIdGenerator <long> sequenceIdGenerator, IIntegrationDataStoreClient integrationDataStoreClient, RequestContextAccessor <IFhirRequestContext> contextAccessor, ILoggerFactory loggerFactory) { EnsureArg.IsNotNull(importResourceLoader, nameof(importResourceLoader)); EnsureArg.IsNotNull(resourceBulkImporter, nameof(resourceBulkImporter)); EnsureArg.IsNotNull(importErrorStoreFactory, nameof(importErrorStoreFactory)); EnsureArg.IsNotNull(importOrchestratorTaskDataStoreOperation, nameof(importOrchestratorTaskDataStoreOperation)); EnsureArg.IsNotNull(contextUpdaterFactory, nameof(contextUpdaterFactory)); EnsureArg.IsNotNull(taskmanager, nameof(taskmanager)); EnsureArg.IsNotNull(sequenceIdGenerator, nameof(sequenceIdGenerator)); EnsureArg.IsNotNull(integrationDataStoreClient, nameof(integrationDataStoreClient)); EnsureArg.IsNotNull(contextAccessor, nameof(contextAccessor)); EnsureArg.IsNotNull(loggerFactory, nameof(loggerFactory)); _importResourceLoader = importResourceLoader; _resourceBulkImporter = resourceBulkImporter; _importErrorStoreFactory = importErrorStoreFactory; _importOrchestratorTaskDataStoreOperation = importOrchestratorTaskDataStoreOperation; _sequenceIdGenerator = sequenceIdGenerator; _integrationDataStoreClient = integrationDataStoreClient; _taskmanager = taskmanager; _contextUpdaterFactory = contextUpdaterFactory; _contextAccessor = contextAccessor; _loggerFactory = loggerFactory; }
public async Task GivenAnOrchestratorTaskAndWrongEtag_WhenOrchestratorTaskStart_ThenTaskShouldFailedWithDetails() { IImportOrchestratorTaskDataStoreOperation fhirDataBulkImportOperation = Substitute.For <IImportOrchestratorTaskDataStoreOperation>(); IContextUpdater contextUpdater = Substitute.For <IContextUpdater>(); RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >(); ILoggerFactory loggerFactory = new NullLoggerFactory(); IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>(); ISequenceIdGenerator <long> sequenceIdGenerator = Substitute.For <ISequenceIdGenerator <long> >(); ImportOrchestratorTaskInputData importOrchestratorTaskInputData = new ImportOrchestratorTaskInputData(); ImportOrchestratorTaskContext importOrchestratorTaskContext = new ImportOrchestratorTaskContext(); ITaskManager taskManager = Substitute.For <ITaskManager>(); importOrchestratorTaskInputData.TaskId = Guid.NewGuid().ToString("N"); importOrchestratorTaskInputData.TaskCreateTime = Clock.UtcNow; importOrchestratorTaskInputData.BaseUri = new Uri("http://dummy"); var inputs = new List <InputResource>(); inputs.Add(new InputResource() { Type = "Resource", Url = new Uri("http://dummy"), Etag = "dummy" }); importOrchestratorTaskInputData.Input = inputs; importOrchestratorTaskInputData.InputFormat = "ndjson"; importOrchestratorTaskInputData.InputSource = new Uri("http://dummy"); importOrchestratorTaskInputData.MaxConcurrentProcessingTaskCount = 1; importOrchestratorTaskInputData.ProcessingTaskQueueId = "default"; importOrchestratorTaskInputData.RequestUri = new Uri("http://dummy"); integrationDataStoreClient.GetPropertiesAsync(Arg.Any <Uri>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { Dictionary <string, object> properties = new Dictionary <string, object>(); properties[IntegrationDataStoreClientConstants.BlobPropertyETag] = "test"; properties[IntegrationDataStoreClientConstants.BlobPropertyLength] = 1000L; return(properties); }); sequenceIdGenerator.GetCurrentSequenceId().Returns(_ => 0L); ImportOrchestratorTask orchestratorTask = new ImportOrchestratorTask( importOrchestratorTaskInputData, importOrchestratorTaskContext, taskManager, sequenceIdGenerator, contextUpdater, contextAccessor, fhirDataBulkImportOperation, integrationDataStoreClient, loggerFactory); TaskResultData result = await orchestratorTask.ExecuteAsync(); ImportTaskErrorResult resultDetails = JsonConvert.DeserializeObject <ImportTaskErrorResult>(result.ResultData); Assert.Equal(TaskResult.Fail, result.Result); Assert.Equal(HttpStatusCode.BadRequest, resultDetails.HttpStatusCode); Assert.NotEmpty(resultDetails.ErrorMessage); }
public ImportErrorStore(IIntegrationDataStoreClient integrationDataStoreClient, Uri fileUri) { EnsureArg.IsNotNull(integrationDataStoreClient, nameof(integrationDataStoreClient)); EnsureArg.IsNotNull(fileUri, nameof(fileUri)); _integrationDataStoreClient = integrationDataStoreClient; _fileUri = fileUri; _recyclableMemoryStreamManager = new RecyclableMemoryStreamManager(); }
public async Task GivenResourceLoader_WhenLoadResourcesWithDifferentResourceType_ThenResourcesWithDifferentTypeShouldBeSkipped() { string errorMessage = "Resource type not match."; using MemoryStream stream = new MemoryStream(); using StreamWriter writer = new StreamWriter(stream); await writer.WriteLineAsync("test"); await writer.FlushAsync(); stream.Position = 0; IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>(); integrationDataStoreClient.DownloadResource(Arg.Any <Uri>(), Arg.Any <long>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(stream); integrationDataStoreClient.TryAcquireLeaseAsync(Arg.Any <Uri>(), Arg.Any <string>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(string.Empty); IImportResourceParser importResourceParser = Substitute.For <IImportResourceParser>(); importResourceParser.Parse(Arg.Any <long>(), Arg.Any <long>(), Arg.Any <string>()) .Returns(callInfo => { ImportResource importResource = new ImportResource(null); return(importResource); }); IImportErrorSerializer serializer = Substitute.For <IImportErrorSerializer>(); serializer.Serialize(Arg.Any <long>(), Arg.Any <Exception>()) .Returns(callInfo => { Exception ex = (Exception)callInfo[1]; return(ex.Message); }); Func <long, long> idGenerator = (i) => i; ImportResourceLoader loader = new ImportResourceLoader(integrationDataStoreClient, importResourceParser, serializer, NullLogger <ImportResourceLoader> .Instance); (Channel <ImportResource> outputChannel, Task importTask) = loader.LoadResources("http://dummy", 0, "DummyType", idGenerator, CancellationToken.None); int errorCount = 0; await foreach (ImportResource resource in outputChannel.Reader.ReadAllAsync()) { Assert.Equal(errorMessage, resource.ImportError); ++errorCount; } await importTask; Assert.Equal(1, errorCount); }
public ImportResourceLoader( IIntegrationDataStoreClient integrationDataStoreClient, IImportResourceParser importResourceParser, IImportErrorSerializer importErrorSerializer, ILogger <ImportResourceLoader> logger) { EnsureArg.IsNotNull(integrationDataStoreClient, nameof(integrationDataStoreClient)); EnsureArg.IsNotNull(importResourceParser, nameof(importResourceParser)); EnsureArg.IsNotNull(importErrorSerializer, nameof(importErrorSerializer)); EnsureArg.IsNotNull(logger, nameof(logger)); _integrationDataStoreClient = integrationDataStoreClient; _importResourceParser = importResourceParser; _importErrorSerializer = importErrorSerializer; _logger = logger; }
private async Task VerifyResourceLoaderAsync(int resourcCount, int batchSize, long startIndex) { long startId = 1; List <string> inputStrings = new List <string>(); using MemoryStream stream = new MemoryStream(); using StreamWriter writer = new StreamWriter(stream); for (int i = 0; i < resourcCount; ++i) { string content = (i + startId).ToString(); inputStrings.Add(content); await writer.WriteLineAsync(content); } await writer.FlushAsync(); stream.Position = 0; IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>(); integrationDataStoreClient.DownloadResource(Arg.Any <Uri>(), Arg.Any <long>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(stream); integrationDataStoreClient.TryAcquireLeaseAsync(Arg.Any <Uri>(), Arg.Any <string>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(string.Empty); IImportResourceParser importResourceParser = Substitute.For <IImportResourceParser>(); importResourceParser.Parse(Arg.Any <long>(), Arg.Any <long>(), Arg.Any <string>()) .Returns(callInfo => { long surrogatedId = (long)callInfo[0]; long index = (long)callInfo[1]; string content = (string)callInfo[2]; ResourceWrapper resourceWrapper = new ResourceWrapper( content, "0", "Dummy", new RawResource(content, Fhir.Core.Models.FhirResourceFormat.Json, true), new ResourceRequest("POST"), DateTimeOffset.UtcNow, false, null, null, null, "SearchParam"); return(new ImportResource(surrogatedId, index, resourceWrapper)); }); IImportErrorSerializer serializer = Substitute.For <IImportErrorSerializer>(); Func <long, long> idGenerator = (i) => startId + i; ImportResourceLoader loader = new ImportResourceLoader(integrationDataStoreClient, importResourceParser, serializer, NullLogger <ImportResourceLoader> .Instance); loader.MaxBatchSize = batchSize; (Channel <ImportResource> outputChannel, Task importTask) = loader.LoadResources("http://dummy", startIndex, null, idGenerator, CancellationToken.None); long currentIndex = startIndex; await foreach (ImportResource resource in outputChannel.Reader.ReadAllAsync()) { string content = idGenerator(currentIndex++).ToString(); Assert.Equal(content, resource.Resource.ResourceId); } await importTask; Assert.Equal(resourcCount, currentIndex); }
public async Task GivenResourceLoader_WhenCancelLoadTask_ThenDataLoadTaskShouldBeCanceled() { string errorMessage = "error"; using MemoryStream stream = new MemoryStream(); using StreamWriter writer = new StreamWriter(stream); await writer.WriteLineAsync("test"); await writer.WriteLineAsync("test"); await writer.WriteLineAsync("test"); await writer.WriteLineAsync("test"); await writer.FlushAsync(); stream.Position = 0; AutoResetEvent resetEvent1 = new AutoResetEvent(false); ManualResetEvent resetEvent2 = new ManualResetEvent(false); IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>(); integrationDataStoreClient.DownloadResource(Arg.Any <Uri>(), Arg.Any <long>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(stream); integrationDataStoreClient.TryAcquireLeaseAsync(Arg.Any <Uri>(), Arg.Any <string>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(string.Empty); IImportResourceParser importResourceParser = Substitute.For <IImportResourceParser>(); importResourceParser.Parse(Arg.Any <long>(), Arg.Any <long>(), Arg.Any <string>()) .Returns(callInfo => { resetEvent1.Set(); resetEvent2.WaitOne(); throw new InvalidCastException(errorMessage); }); IImportErrorSerializer serializer = Substitute.For <IImportErrorSerializer>(); serializer.Serialize(Arg.Any <long>(), Arg.Any <Exception>()) .Returns(callInfo => { Exception ex = (Exception)callInfo[1]; return(ex.Message); }); Func <long, long> idGenerator = (i) => i; ImportResourceLoader loader = new ImportResourceLoader(integrationDataStoreClient, importResourceParser, serializer, NullLogger <ImportResourceLoader> .Instance); CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); (Channel <ImportResource> outputChannel, Task importTask) = loader.LoadResources("http://dummy", 0, null, idGenerator, cancellationTokenSource.Token); resetEvent1.WaitOne(); cancellationTokenSource.Cancel(); resetEvent2.Set(); await foreach (ImportResource resource in outputChannel.Reader.ReadAllAsync()) { // do nothing. } try { await importTask; throw new InvalidOperationException(); } catch (TaskCanceledException) { // Expected error } catch (OperationCanceledException) { // Expected error } }
public ImportErrorStoreFactory(IIntegrationDataStoreClient integrationDataStoreClient) { EnsureArg.IsNotNull(integrationDataStoreClient, nameof(integrationDataStoreClient)); _integrationDataStoreClient = integrationDataStoreClient; }
private static async Task VerifyCommonOrchestratorTaskAsync(int inputFileCount, int concurrentCount, int resumeFrom = -1) { IImportOrchestratorTaskDataStoreOperation fhirDataBulkImportOperation = Substitute.For <IImportOrchestratorTaskDataStoreOperation>(); IContextUpdater contextUpdater = Substitute.For <IContextUpdater>(); RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >(); ILoggerFactory loggerFactory = new NullLoggerFactory(); IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>(); ISequenceIdGenerator <long> sequenceIdGenerator = Substitute.For <ISequenceIdGenerator <long> >(); IMediator mediator = Substitute.For <IMediator>(); ImportOrchestratorTaskInputData importOrchestratorTaskInputData = new ImportOrchestratorTaskInputData(); ImportOrchestratorTaskContext importOrchestratorTaskContext = new ImportOrchestratorTaskContext(); List <(long begin, long end)> surrogatedIdRanges = new List <(long begin, long end)>(); TestTaskManager taskManager = new TestTaskManager(t => { if (t == null) { return(null); } if (t.Status == TaskManagement.TaskStatus.Completed) { return(t); } ImportProcessingTaskInputData processingInput = JsonConvert.DeserializeObject <ImportProcessingTaskInputData>(t.InputData); ImportProcessingTaskResult processingResult = new ImportProcessingTaskResult(); processingResult.ResourceType = processingInput.ResourceType; processingResult.SucceedCount = 1; processingResult.FailedCount = 1; processingResult.ErrorLogLocation = "http://dummy/error"; surrogatedIdRanges.Add((processingInput.BeginSequenceId, processingInput.EndSequenceId)); t.Result = JsonConvert.SerializeObject(new TaskResultData(TaskResult.Success, JsonConvert.SerializeObject(processingResult))); t.Status = TaskManagement.TaskStatus.Completed; return(t); }); importOrchestratorTaskInputData.TaskId = Guid.NewGuid().ToString("N"); importOrchestratorTaskInputData.TaskCreateTime = Clock.UtcNow; importOrchestratorTaskInputData.BaseUri = new Uri("http://dummy"); var inputs = new List <InputResource>(); bool resumeMode = resumeFrom >= 0; for (int i = 0; i < inputFileCount; ++i) { string location = $"http://dummy/{i}"; inputs.Add(new InputResource() { Type = "Resource", Url = new Uri(location) }); if (resumeMode) { if (i <= resumeFrom) { TaskInfo taskInfo = new TaskInfo(); taskInfo.TaskId = Guid.NewGuid().ToString("N"); ImportProcessingTaskResult processingResult = new ImportProcessingTaskResult(); processingResult.ResourceType = "Resource"; processingResult.SucceedCount = 1; processingResult.FailedCount = 1; processingResult.ErrorLogLocation = "http://dummy/error"; taskInfo.Result = JsonConvert.SerializeObject(new TaskResultData(TaskResult.Success, JsonConvert.SerializeObject(processingResult))); taskInfo.Status = TaskManagement.TaskStatus.Completed; await taskManager.CreateTaskAsync(taskInfo, false, CancellationToken.None); importOrchestratorTaskContext.DataProcessingTasks[new Uri(location)] = taskInfo; } else { TaskInfo taskInfo = new TaskInfo(); taskInfo.TaskId = Guid.NewGuid().ToString("N"); ImportProcessingTaskInputData processingInput = new ImportProcessingTaskInputData(); processingInput.BaseUriString = "http://dummy"; processingInput.BeginSequenceId = i; processingInput.EndSequenceId = i + 1; processingInput.ResourceType = "Resource"; taskInfo.InputData = JsonConvert.SerializeObject(processingInput); await taskManager.CreateTaskAsync(taskInfo, false, CancellationToken.None); importOrchestratorTaskContext.DataProcessingTasks[new Uri(location)] = taskInfo; } importOrchestratorTaskContext.Progress = ImportOrchestratorTaskProgress.SubTaskRecordsGenerated; } } importOrchestratorTaskInputData.Input = inputs; importOrchestratorTaskInputData.InputFormat = "ndjson"; importOrchestratorTaskInputData.InputSource = new Uri("http://dummy"); importOrchestratorTaskInputData.MaxConcurrentProcessingTaskCount = concurrentCount; importOrchestratorTaskInputData.ProcessingTaskQueueId = "default"; importOrchestratorTaskInputData.RequestUri = new Uri("http://dummy"); integrationDataStoreClient.GetPropertiesAsync(Arg.Any <Uri>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { Dictionary <string, object> properties = new Dictionary <string, object>(); properties[IntegrationDataStoreClientConstants.BlobPropertyETag] = "test"; properties[IntegrationDataStoreClientConstants.BlobPropertyLength] = 1000L; return(properties); }); sequenceIdGenerator.GetCurrentSequenceId().Returns(_ => 0L); ImportOrchestratorTask orchestratorTask = new ImportOrchestratorTask( mediator, importOrchestratorTaskInputData, importOrchestratorTaskContext, taskManager, sequenceIdGenerator, contextUpdater, contextAccessor, fhirDataBulkImportOperation, integrationDataStoreClient, loggerFactory); orchestratorTask.PollingFrequencyInSeconds = 0; TaskResultData result = await orchestratorTask.ExecuteAsync(); ImportTaskResult resultDetails = JsonConvert.DeserializeObject <ImportTaskResult>(result.ResultData); Assert.Equal(TaskResult.Success, result.Result); Assert.Equal(inputFileCount, resultDetails.Output.Count); foreach (ImportOperationOutcome outcome in resultDetails.Output) { Assert.Equal(1, outcome.Count); Assert.NotNull(outcome.InputUrl); Assert.NotEmpty(outcome.Type); } Assert.Equal(inputFileCount, resultDetails.Error.Count); foreach (ImportFailedOperationOutcome outcome in resultDetails.Error) { Assert.Equal(1, outcome.Count); Assert.NotNull(outcome.InputUrl); Assert.NotEmpty(outcome.Type); Assert.NotEmpty(outcome.Url); } Assert.NotEmpty(resultDetails.Request); Assert.Equal(importOrchestratorTaskInputData.TaskCreateTime, resultDetails.TransactionTime); var orderedSurrogatedIdRanges = surrogatedIdRanges.OrderBy(r => r.begin).ToArray(); Assert.Equal(inputFileCount, orderedSurrogatedIdRanges.Length + resumeFrom + 1); for (int i = 0; i < orderedSurrogatedIdRanges.Length - 1; ++i) { Assert.True(orderedSurrogatedIdRanges[i].end > orderedSurrogatedIdRanges[i].begin); Assert.True(orderedSurrogatedIdRanges[i].end <= orderedSurrogatedIdRanges[i + 1].begin); } _ = mediator.Received().Publish( Arg.Is <ImportTaskMetricsNotification>( notification => notification.Id == importOrchestratorTaskInputData.TaskId && notification.Status == TaskResult.Success.ToString() && notification.CreatedTime == importOrchestratorTaskInputData.TaskCreateTime && notification.SucceedCount == resultDetails.Output.Sum(o => o.Count) && notification.FailedCount == resultDetails.Error.Sum(e => e.Count)), Arg.Any <CancellationToken>()); }
public async Task GivenAnOrchestratorTask_WhenCancelBefore_ThenCanceledResultShouldBeReturn() { IImportOrchestratorTaskDataStoreOperation fhirDataBulkImportOperation = Substitute.For <IImportOrchestratorTaskDataStoreOperation>(); IContextUpdater contextUpdater = Substitute.For <IContextUpdater>(); RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >(); ILoggerFactory loggerFactory = new NullLoggerFactory(); IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>(); ISequenceIdGenerator <long> sequenceIdGenerator = Substitute.For <ISequenceIdGenerator <long> >(); IMediator mediator = Substitute.For <IMediator>(); ImportOrchestratorTaskInputData importOrchestratorTaskInputData = new ImportOrchestratorTaskInputData(); ImportOrchestratorTaskContext importOrchestratorTaskContext = new ImportOrchestratorTaskContext(); List <(long begin, long end)> surrogatedIdRanges = new List <(long begin, long end)>(); TestTaskManager taskManager = new TestTaskManager(t => { if (t == null) { return(null); } ImportProcessingTaskInputData processingInput = JsonConvert.DeserializeObject <ImportProcessingTaskInputData>(t.InputData); ImportProcessingTaskResult processingResult = new ImportProcessingTaskResult(); processingResult.ResourceType = processingInput.ResourceType; processingResult.SucceedCount = 1; processingResult.FailedCount = 1; processingResult.ErrorLogLocation = "http://dummy/error"; surrogatedIdRanges.Add((processingInput.BeginSequenceId, processingInput.EndSequenceId)); t.Result = JsonConvert.SerializeObject(new TaskResultData(TaskResult.Success, JsonConvert.SerializeObject(processingResult))); t.Status = TaskManagement.TaskStatus.Completed; return(t); }); importOrchestratorTaskInputData.TaskId = Guid.NewGuid().ToString("N"); importOrchestratorTaskInputData.TaskCreateTime = Clock.UtcNow; importOrchestratorTaskInputData.BaseUri = new Uri("http://dummy"); var inputs = new List <InputResource>(); inputs.Add(new InputResource() { Type = "Resource", Url = new Uri($"http://dummy") }); importOrchestratorTaskInputData.Input = inputs; importOrchestratorTaskInputData.InputFormat = "ndjson"; importOrchestratorTaskInputData.InputSource = new Uri("http://dummy"); importOrchestratorTaskInputData.MaxConcurrentProcessingTaskCount = 1; importOrchestratorTaskInputData.ProcessingTaskQueueId = "default"; importOrchestratorTaskInputData.RequestUri = new Uri("http://dummy"); integrationDataStoreClient.GetPropertiesAsync(Arg.Any <Uri>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { Dictionary <string, object> properties = new Dictionary <string, object>(); properties[IntegrationDataStoreClientConstants.BlobPropertyETag] = "test"; properties[IntegrationDataStoreClientConstants.BlobPropertyLength] = 1000L; return(properties); }); string latestContext = null; contextUpdater.UpdateContextAsync(Arg.Any <string>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { latestContext = (string)callInfo[0]; return(Task.CompletedTask); }); sequenceIdGenerator.GetCurrentSequenceId().Returns <long>(_ => 0L); ImportOrchestratorTask orchestratorTask = new ImportOrchestratorTask( mediator, importOrchestratorTaskInputData, importOrchestratorTaskContext, taskManager, sequenceIdGenerator, contextUpdater, contextAccessor, fhirDataBulkImportOperation, integrationDataStoreClient, loggerFactory); orchestratorTask.PollingFrequencyInSeconds = 0; orchestratorTask.Cancel(); TaskResultData taskResult = await orchestratorTask.ExecuteAsync(); Assert.Equal(TaskResult.Canceled, taskResult.Result); _ = mediator.Received().Publish( Arg.Is <ImportTaskMetricsNotification>( notification => notification.Id == importOrchestratorTaskInputData.TaskId && notification.Status == TaskResult.Canceled.ToString() && notification.CreatedTime == importOrchestratorTaskInputData.TaskCreateTime && notification.SucceedCount == null && notification.FailedCount == null), Arg.Any <CancellationToken>()); }
public async Task GivenAnOrchestratorTask_WhenIntegrationExceptionThrow_ThenTaskShouldFailedWithDetails() { IImportOrchestratorTaskDataStoreOperation fhirDataBulkImportOperation = Substitute.For <IImportOrchestratorTaskDataStoreOperation>(); IContextUpdater contextUpdater = Substitute.For <IContextUpdater>(); RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >(); ILoggerFactory loggerFactory = new NullLoggerFactory(); IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>(); ISequenceIdGenerator <long> sequenceIdGenerator = Substitute.For <ISequenceIdGenerator <long> >(); ImportOrchestratorTaskInputData importOrchestratorTaskInputData = new ImportOrchestratorTaskInputData(); ImportOrchestratorTaskContext importOrchestratorTaskContext = new ImportOrchestratorTaskContext(); ITaskManager taskManager = Substitute.For <ITaskManager>(); IMediator mediator = Substitute.For <IMediator>(); importOrchestratorTaskInputData.TaskId = Guid.NewGuid().ToString("N"); importOrchestratorTaskInputData.TaskCreateTime = Clock.UtcNow; importOrchestratorTaskInputData.BaseUri = new Uri("http://dummy"); var inputs = new List <InputResource>(); inputs.Add(new InputResource() { Type = "Resource", Url = new Uri("http://dummy"), Etag = "dummy" }); importOrchestratorTaskInputData.Input = inputs; importOrchestratorTaskInputData.InputFormat = "ndjson"; importOrchestratorTaskInputData.InputSource = new Uri("http://dummy"); importOrchestratorTaskInputData.MaxConcurrentProcessingTaskCount = 1; importOrchestratorTaskInputData.ProcessingTaskQueueId = "default"; importOrchestratorTaskInputData.RequestUri = new Uri("http://dummy"); integrationDataStoreClient.GetPropertiesAsync(Arg.Any <Uri>(), Arg.Any <CancellationToken>()) .Returns <Task <Dictionary <string, object> > >(_ => { throw new IntegrationDataStoreException("dummy", HttpStatusCode.Unauthorized); }); sequenceIdGenerator.GetCurrentSequenceId().Returns(_ => 0L); ImportOrchestratorTask orchestratorTask = new ImportOrchestratorTask( mediator, importOrchestratorTaskInputData, importOrchestratorTaskContext, taskManager, sequenceIdGenerator, contextUpdater, contextAccessor, fhirDataBulkImportOperation, integrationDataStoreClient, loggerFactory); TaskResultData result = await orchestratorTask.ExecuteAsync(); ImportTaskErrorResult resultDetails = JsonConvert.DeserializeObject <ImportTaskErrorResult>(result.ResultData); Assert.Equal(TaskResult.Fail, result.Result); Assert.Equal(HttpStatusCode.Unauthorized, resultDetails.HttpStatusCode); Assert.NotEmpty(resultDetails.ErrorMessage); _ = mediator.Received().Publish( Arg.Is <ImportTaskMetricsNotification>( notification => notification.Id == importOrchestratorTaskInputData.TaskId && notification.Status == TaskResult.Fail.ToString() && notification.CreatedTime == importOrchestratorTaskInputData.TaskCreateTime && notification.DataSize == null && notification.SucceedCount == null && notification.FailedCount == null), Arg.Any <CancellationToken>()); }
public async Task GivenAnOrchestratorTask_WhenSubTaskFailed_ThenImportProcessingExceptionShouldBeThrowAndContextUpdated() { IImportOrchestratorTaskDataStoreOperation fhirDataBulkImportOperation = Substitute.For <IImportOrchestratorTaskDataStoreOperation>(); IContextUpdater contextUpdater = Substitute.For <IContextUpdater>(); RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >(); ILoggerFactory loggerFactory = new NullLoggerFactory(); IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>(); ISequenceIdGenerator <long> sequenceIdGenerator = Substitute.For <ISequenceIdGenerator <long> >(); ImportOrchestratorTaskInputData importOrchestratorTaskInputData = new ImportOrchestratorTaskInputData(); ImportOrchestratorTaskContext importOrchestratorTaskContext = new ImportOrchestratorTaskContext(); List <(long begin, long end)> surrogatedIdRanges = new List <(long begin, long end)>(); TestTaskManager taskManager = new TestTaskManager(t => { if (t == null) { return(null); } TaskResultData resultData = new TaskResultData(); resultData.Result = TaskResult.Fail; resultData.ResultData = "error"; t.Result = JsonConvert.SerializeObject(resultData); t.Status = TaskManagement.TaskStatus.Completed; return(t); }); importOrchestratorTaskInputData.TaskId = Guid.NewGuid().ToString("N"); importOrchestratorTaskInputData.TaskCreateTime = Clock.UtcNow; importOrchestratorTaskInputData.BaseUri = new Uri("http://dummy"); var inputs = new List <InputResource>(); inputs.Add(new InputResource() { Type = "Resource", Url = new Uri($"http://dummy") }); importOrchestratorTaskInputData.Input = inputs; importOrchestratorTaskInputData.InputFormat = "ndjson"; importOrchestratorTaskInputData.InputSource = new Uri("http://dummy"); importOrchestratorTaskInputData.MaxConcurrentProcessingTaskCount = 1; importOrchestratorTaskInputData.ProcessingTaskQueueId = "default"; importOrchestratorTaskInputData.RequestUri = new Uri("http://dummy"); integrationDataStoreClient.GetPropertiesAsync(Arg.Any <Uri>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { Dictionary <string, object> properties = new Dictionary <string, object>(); properties[IntegrationDataStoreClientConstants.BlobPropertyETag] = "test"; properties[IntegrationDataStoreClientConstants.BlobPropertyLength] = 1000L; return(properties); }); string latestContext = null; contextUpdater.UpdateContextAsync(Arg.Any <string>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { latestContext = (string)callInfo[0]; return(Task.CompletedTask); }); sequenceIdGenerator.GetCurrentSequenceId().Returns <long>(_ => 0L); ImportOrchestratorTask orchestratorTask = new ImportOrchestratorTask( importOrchestratorTaskInputData, importOrchestratorTaskContext, taskManager, sequenceIdGenerator, contextUpdater, contextAccessor, fhirDataBulkImportOperation, integrationDataStoreClient, loggerFactory); orchestratorTask.PollingFrequencyInSeconds = 0; TaskResultData taskResultData = await orchestratorTask.ExecuteAsync(); Assert.Equal(TaskResult.Fail, taskResultData.Result); ImportOrchestratorTaskContext context = JsonConvert.DeserializeObject <ImportOrchestratorTaskContext>(latestContext); Assert.Equal(ImportOrchestratorTaskProgress.SubTaskRecordsGenerated, context.Progress); }