private async Task <Uri> ImportCheckAsync(ImportRequest request, TestFhirClient client = null, int?errorCount = null)
{
client = client ?? _client;
Uri checkLocation = await ImportTestHelper.CreateImportTaskAsync(client, request);
HttpResponseMessage response;
while ((response = await client.CheckImportAsync(checkLocation, CancellationToken.None)).StatusCode == System.Net.HttpStatusCode.Accepted)
{
await Task.Delay(TimeSpan.FromSeconds(5));
}
Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
ImportTaskResult result = JsonConvert.DeserializeObject <ImportTaskResult>(await response.Content.ReadAsStringAsync());
Assert.NotEmpty(result.Output);
if (errorCount != null)
{
Assert.Equal(errorCount.Value, result.Error.First().Count);
}
else
{
Assert.Empty(result.Error);
}
Assert.NotEmpty(result.Request);
return(checkLocation);
}
public async Task GivenImportOperationEnabled_WhenImportResourceWithWrongType_ThenErrorLogShouldBeUploaded()
{
_metricHandler?.ResetCount();
string patientNdJsonResource = Samples.GetNdJson("Import-Patient");
patientNdJsonResource = Regex.Replace(patientNdJsonResource, "##PatientID##", m => Guid.NewGuid().ToString("N"));
(Uri location, string etag) = await ImportTestHelper.UploadFileAsync(patientNdJsonResource, _fixture.CloudStorageAccount);
var request = new ImportRequest()
{
InputFormat = "application/fhir+ndjson",
InputSource = new Uri("https://other-server.example.org"),
StorageDetail = new ImportRequestStorageDetail()
{
Type = "azure-blob"
},
Input = new List <InputResource>()
{
new InputResource()
{
Url = location,
Etag = etag,
Type = "Observation", // not match the resource
},
},
};
Uri checkLocation = await ImportTestHelper.CreateImportTaskAsync(_client, request);
HttpResponseMessage response;
while ((response = await _client.CheckImportAsync(checkLocation, CancellationToken.None)).StatusCode == System.Net.HttpStatusCode.Accepted)
{
await Task.Delay(TimeSpan.FromSeconds(5));
}
Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
ImportTaskResult result = JsonConvert.DeserializeObject <ImportTaskResult>(await response.Content.ReadAsStringAsync());
Assert.Single(result.Error);
Assert.NotEmpty(result.Error.First().Url);
// Only check metric for local tests
if (_fixture.IsUsingInProcTestServer)
{
var resourceCount = Regex.Matches(patientNdJsonResource, "{\"resourceType\":").Count;
var notificationList = _metricHandler.NotificationMapping[typeof(ImportTaskMetricsNotification)];
Assert.Single(notificationList);
var notification = notificationList.First() as ImportTaskMetricsNotification;
Assert.Equal(TaskResult.Success.ToString(), notification.Status);
Assert.NotNull(notification.DataSize);
Assert.Equal(0, notification.SucceedCount);
Assert.Equal(resourceCount, notification.FailedCount);
}
}
public async Task GivenImportOperationEnabled_WhenImportInvalidResource_ThenErrorLogsShouldBeOutput()
{
string patientNdJsonResource = Samples.GetNdJson("Import-InvalidPatient");
patientNdJsonResource = Regex.Replace(patientNdJsonResource, "##PatientID##", m => Guid.NewGuid().ToString("N"));
(Uri location, string etag) = await ImportTestHelper.UploadFileAsync(patientNdJsonResource, _fixture.CloudStorageAccount);
var request = new ImportRequest()
{
InputFormat = "application/fhir+ndjson",
InputSource = new Uri("https://other-server.example.org"),
StorageDetail = new ImportRequestStorageDetail()
{
Type = "azure-blob"
},
Input = new List <InputResource>()
{
new InputResource()
{
Url = location,
Etag = etag,
Type = "Patient",
},
},
};
Uri checkLocation = await ImportTestHelper.CreateImportTaskAsync(_client, request);
HttpResponseMessage response;
while ((response = await _client.CheckImportAsync(checkLocation, CancellationToken.None)).StatusCode == System.Net.HttpStatusCode.Accepted)
{
await Task.Delay(TimeSpan.FromSeconds(5));
}
Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
ImportTaskResult result = JsonConvert.DeserializeObject <ImportTaskResult>(await response.Content.ReadAsStringAsync());
Assert.NotEmpty(result.Output);
Assert.Equal(1, result.Error.Count);
Assert.NotEmpty(result.Request);
string errorLoation = result.Error.ToArray()[0].Url;
string[] errorContents = (await ImportTestHelper.DownloadFileAsync(errorLoation, _fixture.CloudStorageAccount)).Split("\r\n", StringSplitOptions.RemoveEmptyEntries);
Assert.Single(errorContents);
}
public async Task GivenAFhirMediator_WhenGettingAnExistingBulkImportTaskWithCompletedStatus_ThenHttpResponseCodeShouldBeOk()
{
ImportTaskResult expectedResult = new ImportTaskResult();
expectedResult.Request = "test";
TaskResultData taskResultData = new TaskResultData()
{
Result = TaskResult.Success,
ResultData = JsonConvert.SerializeObject(expectedResult),
};
GetImportResponse result = await SetupAndExecuteGetBulkImportTaskByIdAsync(TaskStatus.Completed, false, taskResultData);
Assert.Equal(HttpStatusCode.OK, result.StatusCode);
Assert.NotNull(result.TaskResult);
}
private static async Task VerifyCommonOrchestratorTaskAsync(int inputFileCount, int concurrentCount, int resumeFrom = -1)
{
IImportOrchestratorTaskDataStoreOperation fhirDataBulkImportOperation = Substitute.For <IImportOrchestratorTaskDataStoreOperation>();
IContextUpdater contextUpdater = Substitute.For <IContextUpdater>();
RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
ILoggerFactory loggerFactory = new NullLoggerFactory();
IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>();
ISequenceIdGenerator <long> sequenceIdGenerator = Substitute.For <ISequenceIdGenerator <long> >();
IMediator mediator = Substitute.For <IMediator>();
ImportOrchestratorTaskInputData importOrchestratorTaskInputData = new ImportOrchestratorTaskInputData();
ImportOrchestratorTaskContext importOrchestratorTaskContext = new ImportOrchestratorTaskContext();
List <(long begin, long end)> surrogatedIdRanges = new List <(long begin, long end)>();
TestTaskManager taskManager = new TestTaskManager(t =>
{
if (t == null)
{
return(null);
}
if (t.Status == TaskManagement.TaskStatus.Completed)
{
return(t);
}
ImportProcessingTaskInputData processingInput = JsonConvert.DeserializeObject <ImportProcessingTaskInputData>(t.InputData);
ImportProcessingTaskResult processingResult = new ImportProcessingTaskResult();
processingResult.ResourceType = processingInput.ResourceType;
processingResult.SucceedCount = 1;
processingResult.FailedCount = 1;
processingResult.ErrorLogLocation = "http://dummy/error";
surrogatedIdRanges.Add((processingInput.BeginSequenceId, processingInput.EndSequenceId));
t.Result = JsonConvert.SerializeObject(new TaskResultData(TaskResult.Success, JsonConvert.SerializeObject(processingResult)));
t.Status = TaskManagement.TaskStatus.Completed;
return(t);
});
importOrchestratorTaskInputData.TaskId = Guid.NewGuid().ToString("N");
importOrchestratorTaskInputData.TaskCreateTime = Clock.UtcNow;
importOrchestratorTaskInputData.BaseUri = new Uri("http://dummy");
var inputs = new List <InputResource>();
bool resumeMode = resumeFrom >= 0;
for (int i = 0; i < inputFileCount; ++i)
{
string location = $"http://dummy/{i}";
inputs.Add(new InputResource()
{
Type = "Resource", Url = new Uri(location)
});
if (resumeMode)
{
if (i <= resumeFrom)
{
TaskInfo taskInfo = new TaskInfo();
taskInfo.TaskId = Guid.NewGuid().ToString("N");
ImportProcessingTaskResult processingResult = new ImportProcessingTaskResult();
processingResult.ResourceType = "Resource";
processingResult.SucceedCount = 1;
processingResult.FailedCount = 1;
processingResult.ErrorLogLocation = "http://dummy/error";
taskInfo.Result = JsonConvert.SerializeObject(new TaskResultData(TaskResult.Success, JsonConvert.SerializeObject(processingResult)));
taskInfo.Status = TaskManagement.TaskStatus.Completed;
await taskManager.CreateTaskAsync(taskInfo, false, CancellationToken.None);
importOrchestratorTaskContext.DataProcessingTasks[new Uri(location)] = taskInfo;
}
else
{
TaskInfo taskInfo = new TaskInfo();
taskInfo.TaskId = Guid.NewGuid().ToString("N");
ImportProcessingTaskInputData processingInput = new ImportProcessingTaskInputData();
processingInput.BaseUriString = "http://dummy";
processingInput.BeginSequenceId = i;
processingInput.EndSequenceId = i + 1;
processingInput.ResourceType = "Resource";
taskInfo.InputData = JsonConvert.SerializeObject(processingInput);
await taskManager.CreateTaskAsync(taskInfo, false, CancellationToken.None);
importOrchestratorTaskContext.DataProcessingTasks[new Uri(location)] = taskInfo;
}
importOrchestratorTaskContext.Progress = ImportOrchestratorTaskProgress.SubTaskRecordsGenerated;
}
}
importOrchestratorTaskInputData.Input = inputs;
importOrchestratorTaskInputData.InputFormat = "ndjson";
importOrchestratorTaskInputData.InputSource = new Uri("http://dummy");
importOrchestratorTaskInputData.MaxConcurrentProcessingTaskCount = concurrentCount;
importOrchestratorTaskInputData.ProcessingTaskQueueId = "default";
importOrchestratorTaskInputData.RequestUri = new Uri("http://dummy");
integrationDataStoreClient.GetPropertiesAsync(Arg.Any <Uri>(), Arg.Any <CancellationToken>())
.Returns(callInfo =>
{
Dictionary <string, object> properties = new Dictionary <string, object>();
properties[IntegrationDataStoreClientConstants.BlobPropertyETag] = "test";
properties[IntegrationDataStoreClientConstants.BlobPropertyLength] = 1000L;
return(properties);
});
sequenceIdGenerator.GetCurrentSequenceId().Returns(_ => 0L);
ImportOrchestratorTask orchestratorTask = new ImportOrchestratorTask(
mediator,
importOrchestratorTaskInputData,
importOrchestratorTaskContext,
taskManager,
sequenceIdGenerator,
contextUpdater,
contextAccessor,
fhirDataBulkImportOperation,
integrationDataStoreClient,
loggerFactory);
orchestratorTask.PollingFrequencyInSeconds = 0;
TaskResultData result = await orchestratorTask.ExecuteAsync();
ImportTaskResult resultDetails = JsonConvert.DeserializeObject <ImportTaskResult>(result.ResultData);
Assert.Equal(TaskResult.Success, result.Result);
Assert.Equal(inputFileCount, resultDetails.Output.Count);
foreach (ImportOperationOutcome outcome in resultDetails.Output)
{
Assert.Equal(1, outcome.Count);
Assert.NotNull(outcome.InputUrl);
Assert.NotEmpty(outcome.Type);
}
Assert.Equal(inputFileCount, resultDetails.Error.Count);
foreach (ImportFailedOperationOutcome outcome in resultDetails.Error)
{
Assert.Equal(1, outcome.Count);
Assert.NotNull(outcome.InputUrl);
Assert.NotEmpty(outcome.Type);
Assert.NotEmpty(outcome.Url);
}
Assert.NotEmpty(resultDetails.Request);
Assert.Equal(importOrchestratorTaskInputData.TaskCreateTime, resultDetails.TransactionTime);
var orderedSurrogatedIdRanges = surrogatedIdRanges.OrderBy(r => r.begin).ToArray();
Assert.Equal(inputFileCount, orderedSurrogatedIdRanges.Length + resumeFrom + 1);
for (int i = 0; i < orderedSurrogatedIdRanges.Length - 1; ++i)
{
Assert.True(orderedSurrogatedIdRanges[i].end > orderedSurrogatedIdRanges[i].begin);
Assert.True(orderedSurrogatedIdRanges[i].end <= orderedSurrogatedIdRanges[i + 1].begin);
}
_ = mediator.Received().Publish(
Arg.Is <ImportTaskMetricsNotification>(
notification => notification.Id == importOrchestratorTaskInputData.TaskId &&
notification.Status == TaskResult.Success.ToString() &&
notification.CreatedTime == importOrchestratorTaskInputData.TaskCreateTime &&
notification.SucceedCount == resultDetails.Output.Sum(o => o.Count) &&
notification.FailedCount == resultDetails.Error.Sum(e => e.Count)),
Arg.Any <CancellationToken>());
}
public GetImportResponse(HttpStatusCode statusCode, ImportTaskResult taskResult)
{
StatusCode = statusCode;
TaskResult = taskResult;
}
