private async Task <Uri> ImportCheckAsync(ImportRequest request, TestFhirClient client = null, int?errorCount = null) { client = client ?? _client; Uri checkLocation = await ImportTestHelper.CreateImportTaskAsync(client, request); HttpResponseMessage response; while ((response = await client.CheckImportAsync(checkLocation, CancellationToken.None)).StatusCode == System.Net.HttpStatusCode.Accepted) { await Task.Delay(TimeSpan.FromSeconds(5)); } Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); ImportTaskResult result = JsonConvert.DeserializeObject <ImportTaskResult>(await response.Content.ReadAsStringAsync()); Assert.NotEmpty(result.Output); if (errorCount != null) { Assert.Equal(errorCount.Value, result.Error.First().Count); } else { Assert.Empty(result.Error); } Assert.NotEmpty(result.Request); return(checkLocation); }
public async Task GivenImportOperationEnabled_WhenImportResourceWithWrongType_ThenErrorLogShouldBeUploaded() { _metricHandler?.ResetCount(); string patientNdJsonResource = Samples.GetNdJson("Import-Patient"); patientNdJsonResource = Regex.Replace(patientNdJsonResource, "##PatientID##", m => Guid.NewGuid().ToString("N")); (Uri location, string etag) = await ImportTestHelper.UploadFileAsync(patientNdJsonResource, _fixture.CloudStorageAccount); var request = new ImportRequest() { InputFormat = "application/fhir+ndjson", InputSource = new Uri("https://other-server.example.org"), StorageDetail = new ImportRequestStorageDetail() { Type = "azure-blob" }, Input = new List <InputResource>() { new InputResource() { Url = location, Etag = etag, Type = "Observation", // not match the resource }, }, }; Uri checkLocation = await ImportTestHelper.CreateImportTaskAsync(_client, request); HttpResponseMessage response; while ((response = await _client.CheckImportAsync(checkLocation, CancellationToken.None)).StatusCode == System.Net.HttpStatusCode.Accepted) { await Task.Delay(TimeSpan.FromSeconds(5)); } Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); ImportTaskResult result = JsonConvert.DeserializeObject <ImportTaskResult>(await response.Content.ReadAsStringAsync()); Assert.Single(result.Error); Assert.NotEmpty(result.Error.First().Url); // Only check metric for local tests if (_fixture.IsUsingInProcTestServer) { var resourceCount = Regex.Matches(patientNdJsonResource, "{\"resourceType\":").Count; var notificationList = _metricHandler.NotificationMapping[typeof(ImportTaskMetricsNotification)]; Assert.Single(notificationList); var notification = notificationList.First() as ImportTaskMetricsNotification; Assert.Equal(TaskResult.Success.ToString(), notification.Status); Assert.NotNull(notification.DataSize); Assert.Equal(0, notification.SucceedCount); Assert.Equal(resourceCount, notification.FailedCount); } }
public async Task GivenImportOperationEnabled_WhenImportInvalidResource_ThenErrorLogsShouldBeOutput() { string patientNdJsonResource = Samples.GetNdJson("Import-InvalidPatient"); patientNdJsonResource = Regex.Replace(patientNdJsonResource, "##PatientID##", m => Guid.NewGuid().ToString("N")); (Uri location, string etag) = await ImportTestHelper.UploadFileAsync(patientNdJsonResource, _fixture.CloudStorageAccount); var request = new ImportRequest() { InputFormat = "application/fhir+ndjson", InputSource = new Uri("https://other-server.example.org"), StorageDetail = new ImportRequestStorageDetail() { Type = "azure-blob" }, Input = new List <InputResource>() { new InputResource() { Url = location, Etag = etag, Type = "Patient", }, }, }; Uri checkLocation = await ImportTestHelper.CreateImportTaskAsync(_client, request); HttpResponseMessage response; while ((response = await _client.CheckImportAsync(checkLocation, CancellationToken.None)).StatusCode == System.Net.HttpStatusCode.Accepted) { await Task.Delay(TimeSpan.FromSeconds(5)); } Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); ImportTaskResult result = JsonConvert.DeserializeObject <ImportTaskResult>(await response.Content.ReadAsStringAsync()); Assert.NotEmpty(result.Output); Assert.Equal(1, result.Error.Count); Assert.NotEmpty(result.Request); string errorLoation = result.Error.ToArray()[0].Url; string[] errorContents = (await ImportTestHelper.DownloadFileAsync(errorLoation, _fixture.CloudStorageAccount)).Split("\r\n", StringSplitOptions.RemoveEmptyEntries); Assert.Single(errorContents); }
public async Task GivenAFhirMediator_WhenGettingAnExistingBulkImportTaskWithCompletedStatus_ThenHttpResponseCodeShouldBeOk() { ImportTaskResult expectedResult = new ImportTaskResult(); expectedResult.Request = "test"; TaskResultData taskResultData = new TaskResultData() { Result = TaskResult.Success, ResultData = JsonConvert.SerializeObject(expectedResult), }; GetImportResponse result = await SetupAndExecuteGetBulkImportTaskByIdAsync(TaskStatus.Completed, false, taskResultData); Assert.Equal(HttpStatusCode.OK, result.StatusCode); Assert.NotNull(result.TaskResult); }
private static async Task VerifyCommonOrchestratorTaskAsync(int inputFileCount, int concurrentCount, int resumeFrom = -1) { IImportOrchestratorTaskDataStoreOperation fhirDataBulkImportOperation = Substitute.For <IImportOrchestratorTaskDataStoreOperation>(); IContextUpdater contextUpdater = Substitute.For <IContextUpdater>(); RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >(); ILoggerFactory loggerFactory = new NullLoggerFactory(); IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>(); ISequenceIdGenerator <long> sequenceIdGenerator = Substitute.For <ISequenceIdGenerator <long> >(); IMediator mediator = Substitute.For <IMediator>(); ImportOrchestratorTaskInputData importOrchestratorTaskInputData = new ImportOrchestratorTaskInputData(); ImportOrchestratorTaskContext importOrchestratorTaskContext = new ImportOrchestratorTaskContext(); List <(long begin, long end)> surrogatedIdRanges = new List <(long begin, long end)>(); TestTaskManager taskManager = new TestTaskManager(t => { if (t == null) { return(null); } if (t.Status == TaskManagement.TaskStatus.Completed) { return(t); } ImportProcessingTaskInputData processingInput = JsonConvert.DeserializeObject <ImportProcessingTaskInputData>(t.InputData); ImportProcessingTaskResult processingResult = new ImportProcessingTaskResult(); processingResult.ResourceType = processingInput.ResourceType; processingResult.SucceedCount = 1; processingResult.FailedCount = 1; processingResult.ErrorLogLocation = "http://dummy/error"; surrogatedIdRanges.Add((processingInput.BeginSequenceId, processingInput.EndSequenceId)); t.Result = JsonConvert.SerializeObject(new TaskResultData(TaskResult.Success, JsonConvert.SerializeObject(processingResult))); t.Status = TaskManagement.TaskStatus.Completed; return(t); }); importOrchestratorTaskInputData.TaskId = Guid.NewGuid().ToString("N"); importOrchestratorTaskInputData.TaskCreateTime = Clock.UtcNow; importOrchestratorTaskInputData.BaseUri = new Uri("http://dummy"); var inputs = new List <InputResource>(); bool resumeMode = resumeFrom >= 0; for (int i = 0; i < inputFileCount; ++i) { string location = $"http://dummy/{i}"; inputs.Add(new InputResource() { Type = "Resource", Url = new Uri(location) }); if (resumeMode) { if (i <= resumeFrom) { TaskInfo taskInfo = new TaskInfo(); taskInfo.TaskId = Guid.NewGuid().ToString("N"); ImportProcessingTaskResult processingResult = new ImportProcessingTaskResult(); processingResult.ResourceType = "Resource"; processingResult.SucceedCount = 1; processingResult.FailedCount = 1; processingResult.ErrorLogLocation = "http://dummy/error"; taskInfo.Result = JsonConvert.SerializeObject(new TaskResultData(TaskResult.Success, JsonConvert.SerializeObject(processingResult))); taskInfo.Status = TaskManagement.TaskStatus.Completed; await taskManager.CreateTaskAsync(taskInfo, false, CancellationToken.None); importOrchestratorTaskContext.DataProcessingTasks[new Uri(location)] = taskInfo; } else { TaskInfo taskInfo = new TaskInfo(); taskInfo.TaskId = Guid.NewGuid().ToString("N"); ImportProcessingTaskInputData processingInput = new ImportProcessingTaskInputData(); processingInput.BaseUriString = "http://dummy"; processingInput.BeginSequenceId = i; processingInput.EndSequenceId = i + 1; processingInput.ResourceType = "Resource"; taskInfo.InputData = JsonConvert.SerializeObject(processingInput); await taskManager.CreateTaskAsync(taskInfo, false, CancellationToken.None); importOrchestratorTaskContext.DataProcessingTasks[new Uri(location)] = taskInfo; } importOrchestratorTaskContext.Progress = ImportOrchestratorTaskProgress.SubTaskRecordsGenerated; } } importOrchestratorTaskInputData.Input = inputs; importOrchestratorTaskInputData.InputFormat = "ndjson"; importOrchestratorTaskInputData.InputSource = new Uri("http://dummy"); importOrchestratorTaskInputData.MaxConcurrentProcessingTaskCount = concurrentCount; importOrchestratorTaskInputData.ProcessingTaskQueueId = "default"; importOrchestratorTaskInputData.RequestUri = new Uri("http://dummy"); integrationDataStoreClient.GetPropertiesAsync(Arg.Any <Uri>(), Arg.Any <CancellationToken>()) .Returns(callInfo => { Dictionary <string, object> properties = new Dictionary <string, object>(); properties[IntegrationDataStoreClientConstants.BlobPropertyETag] = "test"; properties[IntegrationDataStoreClientConstants.BlobPropertyLength] = 1000L; return(properties); }); sequenceIdGenerator.GetCurrentSequenceId().Returns(_ => 0L); ImportOrchestratorTask orchestratorTask = new ImportOrchestratorTask( mediator, importOrchestratorTaskInputData, importOrchestratorTaskContext, taskManager, sequenceIdGenerator, contextUpdater, contextAccessor, fhirDataBulkImportOperation, integrationDataStoreClient, loggerFactory); orchestratorTask.PollingFrequencyInSeconds = 0; TaskResultData result = await orchestratorTask.ExecuteAsync(); ImportTaskResult resultDetails = JsonConvert.DeserializeObject <ImportTaskResult>(result.ResultData); Assert.Equal(TaskResult.Success, result.Result); Assert.Equal(inputFileCount, resultDetails.Output.Count); foreach (ImportOperationOutcome outcome in resultDetails.Output) { Assert.Equal(1, outcome.Count); Assert.NotNull(outcome.InputUrl); Assert.NotEmpty(outcome.Type); } Assert.Equal(inputFileCount, resultDetails.Error.Count); foreach (ImportFailedOperationOutcome outcome in resultDetails.Error) { Assert.Equal(1, outcome.Count); Assert.NotNull(outcome.InputUrl); Assert.NotEmpty(outcome.Type); Assert.NotEmpty(outcome.Url); } Assert.NotEmpty(resultDetails.Request); Assert.Equal(importOrchestratorTaskInputData.TaskCreateTime, resultDetails.TransactionTime); var orderedSurrogatedIdRanges = surrogatedIdRanges.OrderBy(r => r.begin).ToArray(); Assert.Equal(inputFileCount, orderedSurrogatedIdRanges.Length + resumeFrom + 1); for (int i = 0; i < orderedSurrogatedIdRanges.Length - 1; ++i) { Assert.True(orderedSurrogatedIdRanges[i].end > orderedSurrogatedIdRanges[i].begin); Assert.True(orderedSurrogatedIdRanges[i].end <= orderedSurrogatedIdRanges[i + 1].begin); } _ = mediator.Received().Publish( Arg.Is <ImportTaskMetricsNotification>( notification => notification.Id == importOrchestratorTaskInputData.TaskId && notification.Status == TaskResult.Success.ToString() && notification.CreatedTime == importOrchestratorTaskInputData.TaskCreateTime && notification.SucceedCount == resultDetails.Output.Sum(o => o.Count) && notification.FailedCount == resultDetails.Error.Sum(e => e.Count)), Arg.Any <CancellationToken>()); }
public GetImportResponse(HttpStatusCode statusCode, ImportTaskResult taskResult) { StatusCode = statusCode; TaskResult = taskResult; }