public static IFhirServerBuilder AddAzureExportClientInitializer(this IFhirServerBuilder fhirServerBuilder, IConfiguration configuration)
{
EnsureArg.IsNotNull(fhirServerBuilder, nameof(fhirServerBuilder));
EnsureArg.IsNotNull(configuration, nameof(configuration));
var exportJobConfiguration = new ExportJobConfiguration();
configuration.GetSection(ExportConfigurationName).Bind(exportJobConfiguration);
if (!string.IsNullOrWhiteSpace(exportJobConfiguration.StorageAccountUri))
{
fhirServerBuilder.Services.Add <AzureAccessTokenClientInitializer>()
.Transient()
.AsService <IExportClientInitializer <CloudBlobClient> >();
fhirServerBuilder.Services.Add <AzureAccessTokenProvider>()
.Transient()
.AsService <IAccessTokenProvider>();
}
else
{
fhirServerBuilder.Services.Add <AzureConnectionStringClientInitializer>()
.Transient()
.AsService <IExportClientInitializer <CloudBlobClient> >();
}
return(fhirServerBuilder);
}
public AnonymizedExportTests(ExportTestFixture fixture)
{
_isUsingInProcTestServer = fixture.IsUsingInProcTestServer;
_testFhirClient = fixture.TestFhirClient;
_metricHandler = fixture.MetricHandler;
_exportConfiguration = ((IOptions <ExportJobConfiguration>)(fixture.TestFhirServer as InProcTestFhirServer)?.Server?.Services?.GetService(typeof(IOptions <ExportJobConfiguration>)))?.Value;
}
public async Task <CloudBlobClient> GetAuthorizedClientAsync(ExportJobConfiguration exportJobConfiguration, CancellationToken cancellationToken)
{
// Get storage uri from config
if (string.IsNullOrWhiteSpace(exportJobConfiguration.StorageAccountUri))
{
throw new ExportClientInitializerException(Resources.InvalidStorageUri, HttpStatusCode.BadRequest);
}
if (!Uri.TryCreate(exportJobConfiguration.StorageAccountUri, UriKind.Absolute, out Uri storageAccountUri))
{
throw new ExportClientInitializerException(Resources.InvalidStorageUri, HttpStatusCode.BadRequest);
}
string accessToken = null;
try
{
accessToken = await _accessTokenProvider.GetAccessTokenForResourceAsync(storageAccountUri, cancellationToken);
}
catch (AccessTokenProviderException atp)
{
_logger.LogError(atp, "Unable to get access token");
throw new ExportClientInitializerException(Resources.CannotGetAccessToken, HttpStatusCode.Unauthorized);
}
var storageCredentials = new StorageCredentials(new TokenCredential(accessToken));
return(new CloudBlobClient(storageAccountUri, storageCredentials));
}
public ExportJobTask(
IFhirOperationDataStore fhirOperationDataStore,
ISecretStore secretStore,
IOptions <ExportJobConfiguration> exportJobConfiguration,
ISearchService searchService,
IResourceToByteArraySerializer resourceToByteArraySerializer,
IExportDestinationClientFactory exportDestinationClientFactory,
ILogger <ExportJobTask> logger)
{
EnsureArg.IsNotNull(fhirOperationDataStore, nameof(fhirOperationDataStore));
EnsureArg.IsNotNull(secretStore, nameof(secretStore));
EnsureArg.IsNotNull(exportJobConfiguration?.Value, nameof(exportJobConfiguration));
EnsureArg.IsNotNull(searchService, nameof(searchService));
EnsureArg.IsNotNull(resourceToByteArraySerializer, nameof(resourceToByteArraySerializer));
EnsureArg.IsNotNull(exportDestinationClientFactory, nameof(exportDestinationClientFactory));
EnsureArg.IsNotNull(logger, nameof(logger));
_fhirOperationDataStore = fhirOperationDataStore;
_secretStore = secretStore;
_exportJobConfiguration = exportJobConfiguration.Value;
_searchService = searchService;
_resourceToByteArraySerializer = resourceToByteArraySerializer;
_exportDestinationClientFactory = exportDestinationClientFactory;
_logger = logger;
}
public ExportJobTask(
Func <IScoped <IFhirOperationDataStore> > fhirOperationDataStoreFactory,
IOptions <ExportJobConfiguration> exportJobConfiguration,
Func <IScoped <ISearchService> > searchServiceFactory,
IGroupMemberExtractor groupMemberExtractor,
IResourceToByteArraySerializer resourceToByteArraySerializer,
IExportDestinationClient exportDestinationClient,
ILogger <ExportJobTask> logger)
{
EnsureArg.IsNotNull(fhirOperationDataStoreFactory, nameof(fhirOperationDataStoreFactory));
EnsureArg.IsNotNull(exportJobConfiguration?.Value, nameof(exportJobConfiguration));
EnsureArg.IsNotNull(searchServiceFactory, nameof(searchServiceFactory));
EnsureArg.IsNotNull(groupMemberExtractor, nameof(groupMemberExtractor));
EnsureArg.IsNotNull(resourceToByteArraySerializer, nameof(resourceToByteArraySerializer));
EnsureArg.IsNotNull(exportDestinationClient, nameof(exportDestinationClient));
EnsureArg.IsNotNull(logger, nameof(logger));
_fhirOperationDataStoreFactory = fhirOperationDataStoreFactory;
_exportJobConfiguration = exportJobConfiguration.Value;
_searchServiceFactory = searchServiceFactory;
_groupMemberExtractor = groupMemberExtractor;
_resourceToByteArraySerializer = resourceToByteArraySerializer;
_exportDestinationClient = exportDestinationClient;
_logger = logger;
}
public Task <CloudBlobClient> GetAuthorizedClientAsync(ExportJobConfiguration exportJobConfiguration, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(exportJobConfiguration.StorageAccountConnection))
{
throw new ExportClientInitializerException(Resources.InvalidConnectionSettings, HttpStatusCode.BadRequest);
}
if (!CloudStorageAccount.TryParse(exportJobConfiguration.StorageAccountConnection, out CloudStorageAccount cloudAccount))
{
throw new ExportClientInitializerException(Resources.InvalidConnectionSettings, HttpStatusCode.BadRequest);
}
CloudBlobClient blobClient = null;
try
{
blobClient = cloudAccount.CreateCloudBlobClient();
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to create a Cloud Blob Client");
throw new ExportClientInitializerException(Resources.InvalidConnectionSettings, HttpStatusCode.BadRequest);
}
return(Task.FromResult(blobClient));
}
public ExportJobWorkerBackgroundService(ExportJobWorker exportJobWorker, IOptions <ExportJobConfiguration> exportJobConfiguration)
{
EnsureArg.IsNotNull(exportJobWorker, nameof(exportJobWorker));
EnsureArg.IsNotNull(exportJobConfiguration?.Value, nameof(exportJobConfiguration));
_exportJobWorker = exportJobWorker;
_exportJobConfiguration = exportJobConfiguration.Value;
}
public AzureConnectionStringClientInitializer(IOptions <ExportJobConfiguration> exportJobConfiguration, ILogger <AzureConnectionStringClientInitializer> logger)
{
EnsureArg.IsNotNull(exportJobConfiguration?.Value, nameof(exportJobConfiguration));
EnsureArg.IsNotNull(logger, nameof(logger));
_exportJobConfiguration = exportJobConfiguration.Value;
_logger = logger;
}
public ExportDestinationArtifactProvider(
IExportClientInitializer <CloudBlobClient> exportClientInitializer,
IOptions <ExportJobConfiguration> exportJobConfiguration)
{
EnsureArg.IsNotNull(exportClientInitializer, nameof(exportClientInitializer));
EnsureArg.IsNotNull(exportJobConfiguration?.Value, nameof(exportJobConfiguration));
_exportClientInitializer = exportClientInitializer;
_exportJobConfiguration = exportJobConfiguration.Value;
}
public AzureConnectionStringClientInitializerTests()
{
_exportJobConfiguration = new ExportJobConfiguration();
IOptions <ExportJobConfiguration> optionsExportConfig = Substitute.For <IOptions <ExportJobConfiguration> >();
optionsExportConfig.Value.Returns(_exportJobConfiguration);
_logger = Substitute.For <ILogger <AzureConnectionStringClientInitializer> >();
_azureConnectionStringClientInitializer = new AzureConnectionStringClientInitializer(optionsExportConfig, _logger);
}
public AzureAccessTokenClientInitializerTests()
{
_exportJobConfiguration = new ExportJobConfiguration();
IOptions <ExportJobConfiguration> optionsExportConfig = Substitute.For <IOptions <ExportJobConfiguration> >();
optionsExportConfig.Value.Returns(_exportJobConfiguration);
_accessTokenProvider = Substitute.For <IAccessTokenProvider>();
_logger = Substitute.For <ILogger <AzureAccessTokenClientInitializer> >();
_azureAccessTokenClientInitializer = new AzureAccessTokenClientInitializer(_accessTokenProvider, optionsExportConfig, _logger);
}
public ExportJobWorker(Func <IScoped <IFhirOperationDataStore> > fhirOperationDataStoreFactory, IOptions <ExportJobConfiguration> exportJobConfiguration, Func <IExportJobTask> exportJobTaskFactory, ILogger <ExportJobWorker> logger)
{
EnsureArg.IsNotNull(fhirOperationDataStoreFactory, nameof(fhirOperationDataStoreFactory));
EnsureArg.IsNotNull(exportJobConfiguration?.Value, nameof(exportJobConfiguration));
EnsureArg.IsNotNull(exportJobTaskFactory, nameof(exportJobTaskFactory));
EnsureArg.IsNotNull(logger, nameof(logger));
_fhirOperationDataStoreFactory = fhirOperationDataStoreFactory;
_exportJobConfiguration = exportJobConfiguration.Value;
_exportJobTaskFactory = exportJobTaskFactory;
_logger = logger;
}
public CreateExportRequestHandlerTests(FhirStorageTestsFixture fixture)
{
_fhirOperationDataStore = fixture.OperationDataStore;
_fhirStorageTestHelper = fixture.TestHelper;
_exportJobConfiguration = new ExportJobConfiguration();
IOptions <ExportJobConfiguration> optionsExportConfig = Substitute.For <IOptions <ExportJobConfiguration> >();
optionsExportConfig.Value.Returns(_exportJobConfiguration);
_createExportRequestHandler = new CreateExportRequestHandler(_claimsExtractor, _fhirOperationDataStore, DisabledFhirAuthorizationService.Instance, optionsExportConfig);
}
public AzureExportDestinationClientTests()
{
_exportClientInitializer = Substitute.For <IExportClientInitializer <CloudBlobClient> >();
_logger = Substitute.For <ILogger <AzureExportDestinationClient> >();
_exportJobConfiguration = new ExportJobConfiguration();
IOptions <ExportJobConfiguration> optionsExportConfig = Substitute.For <IOptions <ExportJobConfiguration> >();
optionsExportConfig.Value.Returns(_exportJobConfiguration);
_exportDestinationClient = new AzureExportDestinationClient(_exportClientInitializer, optionsExportConfig, _logger);
}
public AzureAccessTokenClientInitializer(
IAccessTokenProvider accessTokenProvider,
IOptions <ExportJobConfiguration> exportJobConfiguration,
ILogger <AzureAccessTokenClientInitializer> logger)
{
EnsureArg.IsNotNull(accessTokenProvider, nameof(accessTokenProvider));
EnsureArg.IsNotNull(exportJobConfiguration?.Value, nameof(exportJobConfiguration));
EnsureArg.IsNotNull(logger, nameof(logger));
_accessTokenProvider = accessTokenProvider;
_exportJobConfiguration = exportJobConfiguration.Value;
_logger = logger;
}
private async Task SearchCompartmentWithFilter(
ExportJobConfiguration exportJobConfiguration,
ExportJobProgress progress,
string resourceType,
List <Tuple <string, string> > queryParametersList,
IAnonymizer anonymizer,
string batchIdPrefix,
CancellationToken cancellationToken)
{
// Current batch will be used to organize a set of search results into a group so that they can be committed together.
string currentBatchId = batchIdPrefix + "-" + progress.Page.ToString("d6");
// Process the export if:
// 1. There is continuation token, which means there is more resource to be exported.
// 2. There is no continuation token but the page is 0, which means it's the initial export.
while (progress.ContinuationToken != null || progress.Page == 0)
{
SearchResult searchResult = null;
// Search and process the results.
using (IScoped <ISearchService> searchService = _searchServiceFactory())
{
searchResult = await searchService.Value.SearchCompartmentAsync(
compartmentType : KnownResourceTypes.Patient,
compartmentId : progress.TriggeringResourceId,
resourceType : resourceType,
queryParametersList,
cancellationToken);
}
await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, anonymizer, cancellationToken);
if (searchResult.ContinuationToken == null)
{
// No more continuation token, we are done.
break;
}
await ProcessProgressChange(exportJobConfiguration, progress, queryParametersList, searchResult.ContinuationToken, false, cancellationToken);
currentBatchId = batchIdPrefix + '-' + progress.Page.ToString("d6");
}
// Commit one last time for any pending changes.
await _exportDestinationClient.CommitAsync(exportJobConfiguration, cancellationToken);
progress.MarkFilterFinished();
await UpdateJobRecordAsync(cancellationToken);
}
public CreateExportRequestHandler(
IClaimsExtractor claimsExtractor,
IFhirOperationDataStore fhirOperationDataStore,
IFhirAuthorizationService authorizationService,
IOptions <ExportJobConfiguration> exportJobConfiguration)
{
EnsureArg.IsNotNull(claimsExtractor, nameof(claimsExtractor));
EnsureArg.IsNotNull(fhirOperationDataStore, nameof(fhirOperationDataStore));
EnsureArg.IsNotNull(authorizationService, nameof(authorizationService));
EnsureArg.IsNotNull(exportJobConfiguration?.Value, nameof(exportJobConfiguration));
_claimsExtractor = claimsExtractor;
_fhirOperationDataStore = fhirOperationDataStore;
_authorizationService = authorizationService;
_exportJobConfiguration = exportJobConfiguration.Value;
}
private async Task ProcessFilterForCompartment(
ExportJobConfiguration exportJobConfiguration,
ExportJobProgress exportJobProgress,
List <Tuple <string, string> > queryParametersList,
string batchIdPrefix,
CancellationToken cancellationToken)
{
var index = _exportJobRecord.Filters.IndexOf(exportJobProgress.CurrentFilter);
List <Tuple <string, string> > filterQueryParametersList = new List <Tuple <string, string> >(queryParametersList);
foreach (var param in exportJobProgress.CurrentFilter.Parameters)
{
filterQueryParametersList.Add(param);
}
await SearchCompartmentWithFilter(exportJobConfiguration, exportJobProgress, exportJobProgress.CurrentFilter.ResourceType, filterQueryParametersList, batchIdPrefix + index, cancellationToken);
}
private ExportController GetController(ExportJobConfiguration exportConfig)
{
var operationConfig = new OperationsConfiguration()
{
Export = exportConfig,
};
IOptions <OperationsConfiguration> optionsOperationConfiguration = Substitute.For <IOptions <OperationsConfiguration> >();
optionsOperationConfiguration.Value.Returns(operationConfig);
return(new ExportController(
_mediator,
_fhirRequestContextAccessor,
_urlResolver,
optionsOperationConfiguration,
NullLogger <ExportController> .Instance));
}
public ExportController(
IMediator mediator,
IFhirRequestContextAccessor fhirRequestContextAccessor,
IUrlResolver urlResolver,
IOptions <OperationsConfiguration> operationsConfig,
ILogger <ExportController> logger)
{
EnsureArg.IsNotNull(mediator, nameof(mediator));
EnsureArg.IsNotNull(fhirRequestContextAccessor, nameof(fhirRequestContextAccessor));
EnsureArg.IsNotNull(urlResolver, nameof(urlResolver));
EnsureArg.IsNotNull(operationsConfig?.Value?.Export, nameof(operationsConfig));
EnsureArg.IsNotNull(logger, nameof(logger));
_mediator = mediator;
_fhirRequestContextAccessor = fhirRequestContextAccessor;
_urlResolver = urlResolver;
_exportConfig = operationsConfig.Value.Export;
_logger = logger;
}
public CreateExportRequestHandlerTests(FhirStorageTestsFixture fixture)
{
_fhirOperationDataStore = AddListener(fixture.OperationDataStore);
_fhirStorageTestHelper = fixture.TestHelper;
_exportJobConfiguration = new ExportJobConfiguration();
_exportJobConfiguration.Formats = new List <ExportJobFormatConfiguration>();
_exportJobConfiguration.Formats.Add(new ExportJobFormatConfiguration()
{
Name = "test",
Format = ExportFormatTags.ResourceName,
});
IOptions <ExportJobConfiguration> optionsExportConfig = Substitute.For <IOptions <ExportJobConfiguration> >();
optionsExportConfig.Value.Returns(_exportJobConfiguration);
_createExportRequestHandler = new CreateExportRequestHandler(_claimsExtractor, _fhirOperationDataStore, DisabledFhirAuthorizationService.Instance, optionsExportConfig);
}
public async Task GivenStorageAccountConnectionDidNotChange_WhenExecuted_ThenJobShouldBeCompleted()
{
ExportJobConfiguration exportJobConfiguration = new ExportJobConfiguration();
exportJobConfiguration.StorageAccountConnection = "connection";
exportJobConfiguration.StorageAccountUri = string.Empty;
var exportJobRecordWithConnection = new ExportJobRecord(
new Uri("https://localhost/ExportJob/"),
"Patient",
"hash",
since: PartialDateTime.MinValue,
storageAccountConnectionHash: Microsoft.Health.Core.Extensions.StringExtensions.ComputeHash(exportJobConfiguration.StorageAccountConnection),
storageAccountUri: exportJobConfiguration.StorageAccountUri);
SetupExportJobRecordAndOperationDataStore(exportJobRecordWithConnection);
var exportJobTask = new ExportJobTask(
() => _fhirOperationDataStore.CreateMockScope(),
Options.Create(exportJobConfiguration),
() => _searchService.CreateMockScope(),
_resourceToByteArraySerializer,
_inMemoryDestinationClient,
NullLogger <ExportJobTask> .Instance);
_searchService.SearchAsync(
Arg.Any <string>(),
Arg.Any <IReadOnlyList <Tuple <string, string> > >(),
_cancellationToken)
.Returns(x => CreateSearchResult());
DateTimeOffset endTimestamp = DateTimeOffset.UtcNow;
using (Mock.Property(() => ClockResolver.UtcNowFunc, () => endTimestamp))
{
await exportJobTask.ExecuteAsync(_exportJobRecord, _weakETag, _cancellationToken);
}
Assert.NotNull(_lastExportJobOutcome);
Assert.Equal(OperationStatus.Completed, _lastExportJobOutcome.JobRecord.Status);
Assert.Equal(endTimestamp, _lastExportJobOutcome.JobRecord.EndTime);
}
private static ValidateExportRequestFilterAttribute GetFilter(ExportJobConfiguration exportJobConfig = null)
{
if (exportJobConfig == null)
{
exportJobConfig = new ExportJobConfiguration();
exportJobConfig.Enabled = true;
exportJobConfig.SupportedDestinations.Add(SupportedDestinationType);
}
var opConfig = new OperationsConfiguration()
{
Export = exportJobConfig,
};
IOptions <OperationsConfiguration> options = Substitute.For <IOptions <OperationsConfiguration> >();
options.Value.Returns(opConfig);
return(new ValidateExportRequestFilterAttribute(options));
}
public async Task GivenStorageAccountUriChanged_WhenExecuted_ThenRecordsAreSentToOldStorageAccount()
{
var exportJobRecordWithChangedConnection = new ExportJobRecord(
new Uri("https://localhost/ExportJob/"),
"Patient",
"hash",
since: PartialDateTime.MinValue,
storageAccountConnectionHash: Microsoft.Health.Core.Extensions.StringExtensions.ComputeHash(_exportJobConfiguration.StorageAccountConnection),
storageAccountUri: "origionalUri");
SetupExportJobRecordAndOperationDataStore(exportJobRecordWithChangedConnection);
ExportJobConfiguration configurationWithUri = new ExportJobConfiguration();
configurationWithUri.StorageAccountUri = "newUri";
IExportDestinationClient mockDestinationClient = Substitute.For <IExportDestinationClient>();
ExportJobConfiguration capturedConfiguration = null;
mockDestinationClient.ConnectAsync(
Arg.Do <ExportJobConfiguration>(arg => capturedConfiguration = arg),
Arg.Any <CancellationToken>(),
Arg.Any <string>())
.Returns(x =>
{
return(Task.CompletedTask);
});
var exportJobTask = new ExportJobTask(
() => _fhirOperationDataStore.CreateMockScope(),
Options.Create(_exportJobConfiguration),
() => _searchService.CreateMockScope(),
_resourceToByteArraySerializer,
mockDestinationClient,
NullLogger <ExportJobTask> .Instance);
await exportJobTask.ExecuteAsync(_exportJobRecord, _weakETag, _cancellationToken);
Assert.Equal(exportJobRecordWithChangedConnection.StorageAccountUri, capturedConfiguration.StorageAccountUri);
}
private async Task ProcessFilter(
ExportJobConfiguration exportJobConfiguration,
ExportJobProgress exportJobProgress,
List <Tuple <string, string> > queryParametersList,
List <Tuple <string, string> > sharedQueryParametersList,
IAnonymizer anonymizer,
string batchIdPrefix,
CancellationToken cancellationToken)
{
var index = _exportJobRecord.Filters.IndexOf(exportJobProgress.CurrentFilter);
List <Tuple <string, string> > filterQueryParametersList = new List <Tuple <string, string> >(queryParametersList);
foreach (var param in exportJobProgress.CurrentFilter.Parameters)
{
filterQueryParametersList.Add(param);
}
await SearchWithFilter(exportJobConfiguration, exportJobProgress, exportJobProgress.CurrentFilter.ResourceType, filterQueryParametersList, sharedQueryParametersList, anonymizer, batchIdPrefix + index + "-", cancellationToken);
exportJobProgress.MarkFilterFinished();
await UpdateJobRecordAsync(cancellationToken);
}
private async Task ProcessProgressChange(
ExportJobConfiguration exportJobConfiguration,
ExportJobProgress progress,
List <Tuple <string, string> > queryParametersList,
string continuationToken,
bool forceCommit,
CancellationToken cancellationToken)
{
// Update the continuation token in local cache and queryParams.
// We will add or udpate the continuation token in the query parameters list.
progress.UpdateContinuationToken(continuationToken);
bool replacedContinuationToken = false;
for (int index = 0; index < queryParametersList.Count; index++)
{
if (queryParametersList[index].Item1 == KnownQueryParameterNames.ContinuationToken)
{
queryParametersList[index] = Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken);
replacedContinuationToken = true;
}
}
if (!replacedContinuationToken)
{
queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken));
}
if (progress.Page % _exportJobRecord.NumberOfPagesPerCommit == 0 || forceCommit)
{
// Commit the changes.
await _exportDestinationClient.CommitAsync(exportJobConfiguration, cancellationToken);
// Update the job record.
await UpdateJobRecordAsync(cancellationToken);
}
}
/// <inheritdoc />
public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETag, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(exportJobRecord, nameof(exportJobRecord));
_exportJobRecord = exportJobRecord;
_weakETag = weakETag;
try
{
ExportJobConfiguration exportJobConfiguration = _exportJobConfiguration;
string connectionHash = string.IsNullOrEmpty(_exportJobConfiguration.StorageAccountConnection) ?
string.Empty :
Microsoft.Health.Core.Extensions.StringExtensions.ComputeHash(_exportJobConfiguration.StorageAccountConnection);
if (string.IsNullOrEmpty(exportJobRecord.StorageAccountUri))
{
if (!string.Equals(exportJobRecord.StorageAccountConnectionHash, connectionHash, StringComparison.Ordinal))
{
throw new DestinationConnectionException("Storage account connection string was updated during an export job.", HttpStatusCode.BadRequest);
}
}
else
{
exportJobConfiguration = new ExportJobConfiguration();
exportJobConfiguration.Enabled = _exportJobConfiguration.Enabled;
exportJobConfiguration.StorageAccountUri = exportJobRecord.StorageAccountUri;
}
// Connect to export destination using appropriate client.
await _exportDestinationClient.ConnectAsync(exportJobConfiguration, cancellationToken, _exportJobRecord.Id);
// If we are resuming a job, we can detect that by checking the progress info from the job record.
// If it is null, then we know we are processing a new job.
if (_exportJobRecord.Progress == null)
{
_exportJobRecord.Progress = new ExportJobProgress(continuationToken: null, page: 0);
}
// The intial list of query parameters will not have a continutation token. We will add that later if we get one back
// from the search result.
var queryParametersList = new List <Tuple <string, string> >()
{
Tuple.Create(KnownQueryParameterNames.Count, _exportJobRecord.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)),
Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{_exportJobRecord.QueuedTime.ToString("o", CultureInfo.InvariantCulture)}"),
};
if (_exportJobRecord.Since != null)
{
queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.LastUpdated, $"ge{_exportJobRecord.Since}"));
}
ExportJobProgress progress = _exportJobRecord.Progress;
await RunExportSearch(exportJobConfiguration, progress, queryParametersList, cancellationToken);
await CompleteJobAsync(OperationStatus.Completed, cancellationToken);
_logger.LogTrace("Successfully completed the job.");
}
catch (JobConflictException)
{
// The export job was updated externally. There might be some additional resources that were exported
// but we will not be updating the job record.
_logger.LogTrace("The job was updated by another process.");
}
catch (DestinationConnectionException dce)
{
_logger.LogError(dce, "Can't connect to destination. The job will be marked as failed.");
_exportJobRecord.FailureDetails = new JobFailureDetails(dce.Message, dce.StatusCode);
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
catch (ResourceNotFoundException rnfe)
{
_logger.LogError(rnfe, "Can't find specified resource. The job will be marked as failed.");
_exportJobRecord.FailureDetails = new JobFailureDetails(rnfe.Message, HttpStatusCode.BadRequest);
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
catch (Exception ex)
{
// The job has encountered an error it cannot recover from.
// Try to update the job to failed state.
_logger.LogError(ex, "Encountered an unhandled exception. The job will be marked as failed.");
_exportJobRecord.FailureDetails = new JobFailureDetails(Resources.UnknownError, HttpStatusCode.InternalServerError);
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
}
private async Task RunExportCompartmentSearch(
ExportJobConfiguration exportJobConfiguration,
ExportJobProgress progress,
List <Tuple <string, string> > sharedQueryParametersList,
CancellationToken cancellationToken,
string batchIdPrefix = "")
{
EnsureArg.IsNotNull(exportJobConfiguration, nameof(exportJobConfiguration));
EnsureArg.IsNotNull(progress, nameof(progress));
EnsureArg.IsNotNull(sharedQueryParametersList, nameof(sharedQueryParametersList));
// Current batch will be used to organize a set of search results into a group so that they can be committed together.
string currentBatchId = batchIdPrefix + "-" + progress.Page.ToString("d6");
List <Tuple <string, string> > queryParametersList = new List <Tuple <string, string> >(sharedQueryParametersList);
if (progress.ContinuationToken != null)
{
queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken));
}
if (!string.IsNullOrEmpty(_exportJobRecord.ResourceType))
{
queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.Type, _exportJobRecord.ResourceType));
}
// Process the export if:
// 1. There is continuation token, which means there is more resource to be exported.
// 2. There is no continuation token but the page is 0, which means it's the initial export.
while (progress.ContinuationToken != null || progress.Page == 0)
{
SearchResult searchResult = null;
// Search and process the results.
using (IScoped <ISearchService> searchService = _searchServiceFactory())
{
searchResult = await searchService.Value.SearchCompartmentAsync(
compartmentType : KnownResourceTypes.Patient,
compartmentId : progress.TriggeringResourceId,
resourceType : null,
queryParametersList,
cancellationToken);
}
await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, cancellationToken);
if (searchResult.ContinuationToken == null)
{
// No more continuation token, we are done.
break;
}
await ProcessProgressChange(exportJobConfiguration, progress, queryParametersList, searchResult.ContinuationToken, false, cancellationToken);
currentBatchId = batchIdPrefix + '-' + progress.Page.ToString("d6");
}
// Commit one last time for any pending changes.
await _exportDestinationClient.CommitAsync(exportJobConfiguration, cancellationToken);
await UpdateJobRecordAsync(cancellationToken);
}
private async Task RunExportSearch(
ExportJobConfiguration exportJobConfiguration,
ExportJobProgress progress,
List <Tuple <string, string> > sharedQueryParametersList,
CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(exportJobConfiguration, nameof(exportJobConfiguration));
EnsureArg.IsNotNull(progress, nameof(progress));
EnsureArg.IsNotNull(sharedQueryParametersList, nameof(sharedQueryParametersList));
// Current batch will be used to organize a set of search results into a group so that they can be committed together.
string currentBatchId = progress.Page.ToString("d6");
List <Tuple <string, string> > queryParametersList = new List <Tuple <string, string> >(sharedQueryParametersList);
if (progress.ContinuationToken != null)
{
queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken));
}
if (_exportJobRecord.ExportType == ExportJobType.Patient)
{
queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.Type, KnownResourceTypes.Patient));
}
else if (_exportJobRecord.ExportType == ExportJobType.All && !string.IsNullOrEmpty(_exportJobRecord.ResourceType))
{
queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.Type, _exportJobRecord.ResourceType));
}
// Process the export if:
// 1. There is continuation token, which means there is more resource to be exported.
// 2. There is no continuation token but the page is 0, which means it's the initial export.
while (progress.ContinuationToken != null || progress.Page == 0)
{
SearchResult searchResult = null;
// Search and process the results.
switch (_exportJobRecord.ExportType)
{
case ExportJobType.All:
case ExportJobType.Patient:
using (IScoped <ISearchService> searchService = _searchServiceFactory())
{
searchResult = await searchService.Value.SearchAsync(
resourceType : null,
queryParametersList,
cancellationToken);
}
break;
case ExportJobType.Group:
searchResult = await GetGroupPatients(
_exportJobRecord.GroupId,
queryParametersList,
_exportJobRecord.QueuedTime,
cancellationToken);
break;
}
if (_exportJobRecord.ExportType == ExportJobType.Patient || _exportJobRecord.ExportType == ExportJobType.Group)
{
uint resultIndex = 0;
foreach (SearchResultEntry result in searchResult.Results)
{
// If a job is resumed in the middle of processing patient compartment resources it will skip patients it has already exported compartment information for.
// This assumes the order of the search results is the same every time the same search is performed.
if (progress.SubSearch != null && result.Resource.ResourceId != progress.SubSearch.TriggeringResourceId)
{
resultIndex++;
continue;
}
if (progress.SubSearch == null)
{
progress.NewSubSearch(result.Resource.ResourceId);
}
await RunExportCompartmentSearch(exportJobConfiguration, progress.SubSearch, sharedQueryParametersList, cancellationToken, currentBatchId + ":" + resultIndex.ToString("d6"));
resultIndex++;
progress.ClearSubSearch();
}
}
await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, cancellationToken);
if (searchResult.ContinuationToken == null)
{
// No more continuation token, we are done.
break;
}
await ProcessProgressChange(
exportJobConfiguration,
progress,
queryParametersList,
searchResult.ContinuationToken,
forceCommit : _exportJobRecord.ExportType == ExportJobType.Patient || _exportJobRecord.ExportType == ExportJobType.Group,
cancellationToken);
currentBatchId = progress.Page.ToString("d6");
}
// Commit one last time for any pending changes.
await _exportDestinationClient.CommitAsync(exportJobConfiguration, cancellationToken);
}
public AnonymizedExportDataValidationTests(HttpIntegrationTestFixture <StartupForAnonymizedExportTestProvider> fixture)
{
_testFhirClient = fixture.TestFhirClient;
_exportConfiguration = ((IOptions <ExportJobConfiguration>)(fixture.TestFhirServer as InProcTestFhirServer).Server.Services.GetService(typeof(IOptions <ExportJobConfiguration>))).Value;
}
