public static IFhirServerBuilder AddAzureExportClientInitializer(this IFhirServerBuilder fhirServerBuilder, IConfiguration configuration) { EnsureArg.IsNotNull(fhirServerBuilder, nameof(fhirServerBuilder)); EnsureArg.IsNotNull(configuration, nameof(configuration)); var exportJobConfiguration = new ExportJobConfiguration(); configuration.GetSection(ExportConfigurationName).Bind(exportJobConfiguration); if (!string.IsNullOrWhiteSpace(exportJobConfiguration.StorageAccountUri)) { fhirServerBuilder.Services.Add <AzureAccessTokenClientInitializer>() .Transient() .AsService <IExportClientInitializer <CloudBlobClient> >(); fhirServerBuilder.Services.Add <AzureAccessTokenProvider>() .Transient() .AsService <IAccessTokenProvider>(); } else { fhirServerBuilder.Services.Add <AzureConnectionStringClientInitializer>() .Transient() .AsService <IExportClientInitializer <CloudBlobClient> >(); } return(fhirServerBuilder); }
public AnonymizedExportTests(ExportTestFixture fixture) { _isUsingInProcTestServer = fixture.IsUsingInProcTestServer; _testFhirClient = fixture.TestFhirClient; _metricHandler = fixture.MetricHandler; _exportConfiguration = ((IOptions <ExportJobConfiguration>)(fixture.TestFhirServer as InProcTestFhirServer)?.Server?.Services?.GetService(typeof(IOptions <ExportJobConfiguration>)))?.Value; }
public async Task <CloudBlobClient> GetAuthorizedClientAsync(ExportJobConfiguration exportJobConfiguration, CancellationToken cancellationToken) { // Get storage uri from config if (string.IsNullOrWhiteSpace(exportJobConfiguration.StorageAccountUri)) { throw new ExportClientInitializerException(Resources.InvalidStorageUri, HttpStatusCode.BadRequest); } if (!Uri.TryCreate(exportJobConfiguration.StorageAccountUri, UriKind.Absolute, out Uri storageAccountUri)) { throw new ExportClientInitializerException(Resources.InvalidStorageUri, HttpStatusCode.BadRequest); } string accessToken = null; try { accessToken = await _accessTokenProvider.GetAccessTokenForResourceAsync(storageAccountUri, cancellationToken); } catch (AccessTokenProviderException atp) { _logger.LogError(atp, "Unable to get access token"); throw new ExportClientInitializerException(Resources.CannotGetAccessToken, HttpStatusCode.Unauthorized); } var storageCredentials = new StorageCredentials(new TokenCredential(accessToken)); return(new CloudBlobClient(storageAccountUri, storageCredentials)); }
public ExportJobTask( IFhirOperationDataStore fhirOperationDataStore, ISecretStore secretStore, IOptions <ExportJobConfiguration> exportJobConfiguration, ISearchService searchService, IResourceToByteArraySerializer resourceToByteArraySerializer, IExportDestinationClientFactory exportDestinationClientFactory, ILogger <ExportJobTask> logger) { EnsureArg.IsNotNull(fhirOperationDataStore, nameof(fhirOperationDataStore)); EnsureArg.IsNotNull(secretStore, nameof(secretStore)); EnsureArg.IsNotNull(exportJobConfiguration?.Value, nameof(exportJobConfiguration)); EnsureArg.IsNotNull(searchService, nameof(searchService)); EnsureArg.IsNotNull(resourceToByteArraySerializer, nameof(resourceToByteArraySerializer)); EnsureArg.IsNotNull(exportDestinationClientFactory, nameof(exportDestinationClientFactory)); EnsureArg.IsNotNull(logger, nameof(logger)); _fhirOperationDataStore = fhirOperationDataStore; _secretStore = secretStore; _exportJobConfiguration = exportJobConfiguration.Value; _searchService = searchService; _resourceToByteArraySerializer = resourceToByteArraySerializer; _exportDestinationClientFactory = exportDestinationClientFactory; _logger = logger; }
public ExportJobTask( Func <IScoped <IFhirOperationDataStore> > fhirOperationDataStoreFactory, IOptions <ExportJobConfiguration> exportJobConfiguration, Func <IScoped <ISearchService> > searchServiceFactory, IGroupMemberExtractor groupMemberExtractor, IResourceToByteArraySerializer resourceToByteArraySerializer, IExportDestinationClient exportDestinationClient, ILogger <ExportJobTask> logger) { EnsureArg.IsNotNull(fhirOperationDataStoreFactory, nameof(fhirOperationDataStoreFactory)); EnsureArg.IsNotNull(exportJobConfiguration?.Value, nameof(exportJobConfiguration)); EnsureArg.IsNotNull(searchServiceFactory, nameof(searchServiceFactory)); EnsureArg.IsNotNull(groupMemberExtractor, nameof(groupMemberExtractor)); EnsureArg.IsNotNull(resourceToByteArraySerializer, nameof(resourceToByteArraySerializer)); EnsureArg.IsNotNull(exportDestinationClient, nameof(exportDestinationClient)); EnsureArg.IsNotNull(logger, nameof(logger)); _fhirOperationDataStoreFactory = fhirOperationDataStoreFactory; _exportJobConfiguration = exportJobConfiguration.Value; _searchServiceFactory = searchServiceFactory; _groupMemberExtractor = groupMemberExtractor; _resourceToByteArraySerializer = resourceToByteArraySerializer; _exportDestinationClient = exportDestinationClient; _logger = logger; }
public Task <CloudBlobClient> GetAuthorizedClientAsync(ExportJobConfiguration exportJobConfiguration, CancellationToken cancellationToken) { if (string.IsNullOrWhiteSpace(exportJobConfiguration.StorageAccountConnection)) { throw new ExportClientInitializerException(Resources.InvalidConnectionSettings, HttpStatusCode.BadRequest); } if (!CloudStorageAccount.TryParse(exportJobConfiguration.StorageAccountConnection, out CloudStorageAccount cloudAccount)) { throw new ExportClientInitializerException(Resources.InvalidConnectionSettings, HttpStatusCode.BadRequest); } CloudBlobClient blobClient = null; try { blobClient = cloudAccount.CreateCloudBlobClient(); } catch (Exception ex) { _logger.LogError(ex, "Failed to create a Cloud Blob Client"); throw new ExportClientInitializerException(Resources.InvalidConnectionSettings, HttpStatusCode.BadRequest); } return(Task.FromResult(blobClient)); }
public ExportJobWorkerBackgroundService(ExportJobWorker exportJobWorker, IOptions <ExportJobConfiguration> exportJobConfiguration) { EnsureArg.IsNotNull(exportJobWorker, nameof(exportJobWorker)); EnsureArg.IsNotNull(exportJobConfiguration?.Value, nameof(exportJobConfiguration)); _exportJobWorker = exportJobWorker; _exportJobConfiguration = exportJobConfiguration.Value; }
public AzureConnectionStringClientInitializer(IOptions <ExportJobConfiguration> exportJobConfiguration, ILogger <AzureConnectionStringClientInitializer> logger) { EnsureArg.IsNotNull(exportJobConfiguration?.Value, nameof(exportJobConfiguration)); EnsureArg.IsNotNull(logger, nameof(logger)); _exportJobConfiguration = exportJobConfiguration.Value; _logger = logger; }
public ExportDestinationArtifactProvider( IExportClientInitializer <CloudBlobClient> exportClientInitializer, IOptions <ExportJobConfiguration> exportJobConfiguration) { EnsureArg.IsNotNull(exportClientInitializer, nameof(exportClientInitializer)); EnsureArg.IsNotNull(exportJobConfiguration?.Value, nameof(exportJobConfiguration)); _exportClientInitializer = exportClientInitializer; _exportJobConfiguration = exportJobConfiguration.Value; }
public AzureConnectionStringClientInitializerTests() { _exportJobConfiguration = new ExportJobConfiguration(); IOptions <ExportJobConfiguration> optionsExportConfig = Substitute.For <IOptions <ExportJobConfiguration> >(); optionsExportConfig.Value.Returns(_exportJobConfiguration); _logger = Substitute.For <ILogger <AzureConnectionStringClientInitializer> >(); _azureConnectionStringClientInitializer = new AzureConnectionStringClientInitializer(optionsExportConfig, _logger); }
public AzureAccessTokenClientInitializerTests() { _exportJobConfiguration = new ExportJobConfiguration(); IOptions <ExportJobConfiguration> optionsExportConfig = Substitute.For <IOptions <ExportJobConfiguration> >(); optionsExportConfig.Value.Returns(_exportJobConfiguration); _accessTokenProvider = Substitute.For <IAccessTokenProvider>(); _logger = Substitute.For <ILogger <AzureAccessTokenClientInitializer> >(); _azureAccessTokenClientInitializer = new AzureAccessTokenClientInitializer(_accessTokenProvider, optionsExportConfig, _logger); }
public ExportJobWorker(Func <IScoped <IFhirOperationDataStore> > fhirOperationDataStoreFactory, IOptions <ExportJobConfiguration> exportJobConfiguration, Func <IExportJobTask> exportJobTaskFactory, ILogger <ExportJobWorker> logger) { EnsureArg.IsNotNull(fhirOperationDataStoreFactory, nameof(fhirOperationDataStoreFactory)); EnsureArg.IsNotNull(exportJobConfiguration?.Value, nameof(exportJobConfiguration)); EnsureArg.IsNotNull(exportJobTaskFactory, nameof(exportJobTaskFactory)); EnsureArg.IsNotNull(logger, nameof(logger)); _fhirOperationDataStoreFactory = fhirOperationDataStoreFactory; _exportJobConfiguration = exportJobConfiguration.Value; _exportJobTaskFactory = exportJobTaskFactory; _logger = logger; }
public CreateExportRequestHandlerTests(FhirStorageTestsFixture fixture) { _fhirOperationDataStore = fixture.OperationDataStore; _fhirStorageTestHelper = fixture.TestHelper; _exportJobConfiguration = new ExportJobConfiguration(); IOptions <ExportJobConfiguration> optionsExportConfig = Substitute.For <IOptions <ExportJobConfiguration> >(); optionsExportConfig.Value.Returns(_exportJobConfiguration); _createExportRequestHandler = new CreateExportRequestHandler(_claimsExtractor, _fhirOperationDataStore, DisabledFhirAuthorizationService.Instance, optionsExportConfig); }
public AzureExportDestinationClientTests() { _exportClientInitializer = Substitute.For <IExportClientInitializer <CloudBlobClient> >(); _logger = Substitute.For <ILogger <AzureExportDestinationClient> >(); _exportJobConfiguration = new ExportJobConfiguration(); IOptions <ExportJobConfiguration> optionsExportConfig = Substitute.For <IOptions <ExportJobConfiguration> >(); optionsExportConfig.Value.Returns(_exportJobConfiguration); _exportDestinationClient = new AzureExportDestinationClient(_exportClientInitializer, optionsExportConfig, _logger); }
public AzureAccessTokenClientInitializer( IAccessTokenProvider accessTokenProvider, IOptions <ExportJobConfiguration> exportJobConfiguration, ILogger <AzureAccessTokenClientInitializer> logger) { EnsureArg.IsNotNull(accessTokenProvider, nameof(accessTokenProvider)); EnsureArg.IsNotNull(exportJobConfiguration?.Value, nameof(exportJobConfiguration)); EnsureArg.IsNotNull(logger, nameof(logger)); _accessTokenProvider = accessTokenProvider; _exportJobConfiguration = exportJobConfiguration.Value; _logger = logger; }
private async Task SearchCompartmentWithFilter( ExportJobConfiguration exportJobConfiguration, ExportJobProgress progress, string resourceType, List <Tuple <string, string> > queryParametersList, IAnonymizer anonymizer, string batchIdPrefix, CancellationToken cancellationToken) { // Current batch will be used to organize a set of search results into a group so that they can be committed together. string currentBatchId = batchIdPrefix + "-" + progress.Page.ToString("d6"); // Process the export if: // 1. There is continuation token, which means there is more resource to be exported. // 2. There is no continuation token but the page is 0, which means it's the initial export. while (progress.ContinuationToken != null || progress.Page == 0) { SearchResult searchResult = null; // Search and process the results. using (IScoped <ISearchService> searchService = _searchServiceFactory()) { searchResult = await searchService.Value.SearchCompartmentAsync( compartmentType : KnownResourceTypes.Patient, compartmentId : progress.TriggeringResourceId, resourceType : resourceType, queryParametersList, cancellationToken); } await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, anonymizer, cancellationToken); if (searchResult.ContinuationToken == null) { // No more continuation token, we are done. break; } await ProcessProgressChange(exportJobConfiguration, progress, queryParametersList, searchResult.ContinuationToken, false, cancellationToken); currentBatchId = batchIdPrefix + '-' + progress.Page.ToString("d6"); } // Commit one last time for any pending changes. await _exportDestinationClient.CommitAsync(exportJobConfiguration, cancellationToken); progress.MarkFilterFinished(); await UpdateJobRecordAsync(cancellationToken); }
public CreateExportRequestHandler( IClaimsExtractor claimsExtractor, IFhirOperationDataStore fhirOperationDataStore, IFhirAuthorizationService authorizationService, IOptions <ExportJobConfiguration> exportJobConfiguration) { EnsureArg.IsNotNull(claimsExtractor, nameof(claimsExtractor)); EnsureArg.IsNotNull(fhirOperationDataStore, nameof(fhirOperationDataStore)); EnsureArg.IsNotNull(authorizationService, nameof(authorizationService)); EnsureArg.IsNotNull(exportJobConfiguration?.Value, nameof(exportJobConfiguration)); _claimsExtractor = claimsExtractor; _fhirOperationDataStore = fhirOperationDataStore; _authorizationService = authorizationService; _exportJobConfiguration = exportJobConfiguration.Value; }
private async Task ProcessFilterForCompartment( ExportJobConfiguration exportJobConfiguration, ExportJobProgress exportJobProgress, List <Tuple <string, string> > queryParametersList, string batchIdPrefix, CancellationToken cancellationToken) { var index = _exportJobRecord.Filters.IndexOf(exportJobProgress.CurrentFilter); List <Tuple <string, string> > filterQueryParametersList = new List <Tuple <string, string> >(queryParametersList); foreach (var param in exportJobProgress.CurrentFilter.Parameters) { filterQueryParametersList.Add(param); } await SearchCompartmentWithFilter(exportJobConfiguration, exportJobProgress, exportJobProgress.CurrentFilter.ResourceType, filterQueryParametersList, batchIdPrefix + index, cancellationToken); }
private ExportController GetController(ExportJobConfiguration exportConfig) { var operationConfig = new OperationsConfiguration() { Export = exportConfig, }; IOptions <OperationsConfiguration> optionsOperationConfiguration = Substitute.For <IOptions <OperationsConfiguration> >(); optionsOperationConfiguration.Value.Returns(operationConfig); return(new ExportController( _mediator, _fhirRequestContextAccessor, _urlResolver, optionsOperationConfiguration, NullLogger <ExportController> .Instance)); }
public ExportController( IMediator mediator, IFhirRequestContextAccessor fhirRequestContextAccessor, IUrlResolver urlResolver, IOptions <OperationsConfiguration> operationsConfig, ILogger <ExportController> logger) { EnsureArg.IsNotNull(mediator, nameof(mediator)); EnsureArg.IsNotNull(fhirRequestContextAccessor, nameof(fhirRequestContextAccessor)); EnsureArg.IsNotNull(urlResolver, nameof(urlResolver)); EnsureArg.IsNotNull(operationsConfig?.Value?.Export, nameof(operationsConfig)); EnsureArg.IsNotNull(logger, nameof(logger)); _mediator = mediator; _fhirRequestContextAccessor = fhirRequestContextAccessor; _urlResolver = urlResolver; _exportConfig = operationsConfig.Value.Export; _logger = logger; }
public CreateExportRequestHandlerTests(FhirStorageTestsFixture fixture) { _fhirOperationDataStore = AddListener(fixture.OperationDataStore); _fhirStorageTestHelper = fixture.TestHelper; _exportJobConfiguration = new ExportJobConfiguration(); _exportJobConfiguration.Formats = new List <ExportJobFormatConfiguration>(); _exportJobConfiguration.Formats.Add(new ExportJobFormatConfiguration() { Name = "test", Format = ExportFormatTags.ResourceName, }); IOptions <ExportJobConfiguration> optionsExportConfig = Substitute.For <IOptions <ExportJobConfiguration> >(); optionsExportConfig.Value.Returns(_exportJobConfiguration); _createExportRequestHandler = new CreateExportRequestHandler(_claimsExtractor, _fhirOperationDataStore, DisabledFhirAuthorizationService.Instance, optionsExportConfig); }
public async Task GivenStorageAccountConnectionDidNotChange_WhenExecuted_ThenJobShouldBeCompleted() { ExportJobConfiguration exportJobConfiguration = new ExportJobConfiguration(); exportJobConfiguration.StorageAccountConnection = "connection"; exportJobConfiguration.StorageAccountUri = string.Empty; var exportJobRecordWithConnection = new ExportJobRecord( new Uri("https://localhost/ExportJob/"), "Patient", "hash", since: PartialDateTime.MinValue, storageAccountConnectionHash: Microsoft.Health.Core.Extensions.StringExtensions.ComputeHash(exportJobConfiguration.StorageAccountConnection), storageAccountUri: exportJobConfiguration.StorageAccountUri); SetupExportJobRecordAndOperationDataStore(exportJobRecordWithConnection); var exportJobTask = new ExportJobTask( () => _fhirOperationDataStore.CreateMockScope(), Options.Create(exportJobConfiguration), () => _searchService.CreateMockScope(), _resourceToByteArraySerializer, _inMemoryDestinationClient, NullLogger <ExportJobTask> .Instance); _searchService.SearchAsync( Arg.Any <string>(), Arg.Any <IReadOnlyList <Tuple <string, string> > >(), _cancellationToken) .Returns(x => CreateSearchResult()); DateTimeOffset endTimestamp = DateTimeOffset.UtcNow; using (Mock.Property(() => ClockResolver.UtcNowFunc, () => endTimestamp)) { await exportJobTask.ExecuteAsync(_exportJobRecord, _weakETag, _cancellationToken); } Assert.NotNull(_lastExportJobOutcome); Assert.Equal(OperationStatus.Completed, _lastExportJobOutcome.JobRecord.Status); Assert.Equal(endTimestamp, _lastExportJobOutcome.JobRecord.EndTime); }
private static ValidateExportRequestFilterAttribute GetFilter(ExportJobConfiguration exportJobConfig = null) { if (exportJobConfig == null) { exportJobConfig = new ExportJobConfiguration(); exportJobConfig.Enabled = true; exportJobConfig.SupportedDestinations.Add(SupportedDestinationType); } var opConfig = new OperationsConfiguration() { Export = exportJobConfig, }; IOptions <OperationsConfiguration> options = Substitute.For <IOptions <OperationsConfiguration> >(); options.Value.Returns(opConfig); return(new ValidateExportRequestFilterAttribute(options)); }
public async Task GivenStorageAccountUriChanged_WhenExecuted_ThenRecordsAreSentToOldStorageAccount() { var exportJobRecordWithChangedConnection = new ExportJobRecord( new Uri("https://localhost/ExportJob/"), "Patient", "hash", since: PartialDateTime.MinValue, storageAccountConnectionHash: Microsoft.Health.Core.Extensions.StringExtensions.ComputeHash(_exportJobConfiguration.StorageAccountConnection), storageAccountUri: "origionalUri"); SetupExportJobRecordAndOperationDataStore(exportJobRecordWithChangedConnection); ExportJobConfiguration configurationWithUri = new ExportJobConfiguration(); configurationWithUri.StorageAccountUri = "newUri"; IExportDestinationClient mockDestinationClient = Substitute.For <IExportDestinationClient>(); ExportJobConfiguration capturedConfiguration = null; mockDestinationClient.ConnectAsync( Arg.Do <ExportJobConfiguration>(arg => capturedConfiguration = arg), Arg.Any <CancellationToken>(), Arg.Any <string>()) .Returns(x => { return(Task.CompletedTask); }); var exportJobTask = new ExportJobTask( () => _fhirOperationDataStore.CreateMockScope(), Options.Create(_exportJobConfiguration), () => _searchService.CreateMockScope(), _resourceToByteArraySerializer, mockDestinationClient, NullLogger <ExportJobTask> .Instance); await exportJobTask.ExecuteAsync(_exportJobRecord, _weakETag, _cancellationToken); Assert.Equal(exportJobRecordWithChangedConnection.StorageAccountUri, capturedConfiguration.StorageAccountUri); }
private async Task ProcessFilter( ExportJobConfiguration exportJobConfiguration, ExportJobProgress exportJobProgress, List <Tuple <string, string> > queryParametersList, List <Tuple <string, string> > sharedQueryParametersList, IAnonymizer anonymizer, string batchIdPrefix, CancellationToken cancellationToken) { var index = _exportJobRecord.Filters.IndexOf(exportJobProgress.CurrentFilter); List <Tuple <string, string> > filterQueryParametersList = new List <Tuple <string, string> >(queryParametersList); foreach (var param in exportJobProgress.CurrentFilter.Parameters) { filterQueryParametersList.Add(param); } await SearchWithFilter(exportJobConfiguration, exportJobProgress, exportJobProgress.CurrentFilter.ResourceType, filterQueryParametersList, sharedQueryParametersList, anonymizer, batchIdPrefix + index + "-", cancellationToken); exportJobProgress.MarkFilterFinished(); await UpdateJobRecordAsync(cancellationToken); }
private async Task ProcessProgressChange( ExportJobConfiguration exportJobConfiguration, ExportJobProgress progress, List <Tuple <string, string> > queryParametersList, string continuationToken, bool forceCommit, CancellationToken cancellationToken) { // Update the continuation token in local cache and queryParams. // We will add or udpate the continuation token in the query parameters list. progress.UpdateContinuationToken(continuationToken); bool replacedContinuationToken = false; for (int index = 0; index < queryParametersList.Count; index++) { if (queryParametersList[index].Item1 == KnownQueryParameterNames.ContinuationToken) { queryParametersList[index] = Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken); replacedContinuationToken = true; } } if (!replacedContinuationToken) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken)); } if (progress.Page % _exportJobRecord.NumberOfPagesPerCommit == 0 || forceCommit) { // Commit the changes. await _exportDestinationClient.CommitAsync(exportJobConfiguration, cancellationToken); // Update the job record. await UpdateJobRecordAsync(cancellationToken); } }
/// <inheritdoc /> public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETag, CancellationToken cancellationToken) { EnsureArg.IsNotNull(exportJobRecord, nameof(exportJobRecord)); _exportJobRecord = exportJobRecord; _weakETag = weakETag; try { ExportJobConfiguration exportJobConfiguration = _exportJobConfiguration; string connectionHash = string.IsNullOrEmpty(_exportJobConfiguration.StorageAccountConnection) ? string.Empty : Microsoft.Health.Core.Extensions.StringExtensions.ComputeHash(_exportJobConfiguration.StorageAccountConnection); if (string.IsNullOrEmpty(exportJobRecord.StorageAccountUri)) { if (!string.Equals(exportJobRecord.StorageAccountConnectionHash, connectionHash, StringComparison.Ordinal)) { throw new DestinationConnectionException("Storage account connection string was updated during an export job.", HttpStatusCode.BadRequest); } } else { exportJobConfiguration = new ExportJobConfiguration(); exportJobConfiguration.Enabled = _exportJobConfiguration.Enabled; exportJobConfiguration.StorageAccountUri = exportJobRecord.StorageAccountUri; } // Connect to export destination using appropriate client. await _exportDestinationClient.ConnectAsync(exportJobConfiguration, cancellationToken, _exportJobRecord.Id); // If we are resuming a job, we can detect that by checking the progress info from the job record. // If it is null, then we know we are processing a new job. if (_exportJobRecord.Progress == null) { _exportJobRecord.Progress = new ExportJobProgress(continuationToken: null, page: 0); } // The intial list of query parameters will not have a continutation token. We will add that later if we get one back // from the search result. var queryParametersList = new List <Tuple <string, string> >() { Tuple.Create(KnownQueryParameterNames.Count, _exportJobRecord.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)), Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{_exportJobRecord.QueuedTime.ToString("o", CultureInfo.InvariantCulture)}"), }; if (_exportJobRecord.Since != null) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.LastUpdated, $"ge{_exportJobRecord.Since}")); } ExportJobProgress progress = _exportJobRecord.Progress; await RunExportSearch(exportJobConfiguration, progress, queryParametersList, cancellationToken); await CompleteJobAsync(OperationStatus.Completed, cancellationToken); _logger.LogTrace("Successfully completed the job."); } catch (JobConflictException) { // The export job was updated externally. There might be some additional resources that were exported // but we will not be updating the job record. _logger.LogTrace("The job was updated by another process."); } catch (DestinationConnectionException dce) { _logger.LogError(dce, "Can't connect to destination. The job will be marked as failed."); _exportJobRecord.FailureDetails = new JobFailureDetails(dce.Message, dce.StatusCode); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } catch (ResourceNotFoundException rnfe) { _logger.LogError(rnfe, "Can't find specified resource. The job will be marked as failed."); _exportJobRecord.FailureDetails = new JobFailureDetails(rnfe.Message, HttpStatusCode.BadRequest); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } catch (Exception ex) { // The job has encountered an error it cannot recover from. // Try to update the job to failed state. _logger.LogError(ex, "Encountered an unhandled exception. The job will be marked as failed."); _exportJobRecord.FailureDetails = new JobFailureDetails(Resources.UnknownError, HttpStatusCode.InternalServerError); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } }
private async Task RunExportCompartmentSearch( ExportJobConfiguration exportJobConfiguration, ExportJobProgress progress, List <Tuple <string, string> > sharedQueryParametersList, CancellationToken cancellationToken, string batchIdPrefix = "") { EnsureArg.IsNotNull(exportJobConfiguration, nameof(exportJobConfiguration)); EnsureArg.IsNotNull(progress, nameof(progress)); EnsureArg.IsNotNull(sharedQueryParametersList, nameof(sharedQueryParametersList)); // Current batch will be used to organize a set of search results into a group so that they can be committed together. string currentBatchId = batchIdPrefix + "-" + progress.Page.ToString("d6"); List <Tuple <string, string> > queryParametersList = new List <Tuple <string, string> >(sharedQueryParametersList); if (progress.ContinuationToken != null) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken)); } if (!string.IsNullOrEmpty(_exportJobRecord.ResourceType)) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.Type, _exportJobRecord.ResourceType)); } // Process the export if: // 1. There is continuation token, which means there is more resource to be exported. // 2. There is no continuation token but the page is 0, which means it's the initial export. while (progress.ContinuationToken != null || progress.Page == 0) { SearchResult searchResult = null; // Search and process the results. using (IScoped <ISearchService> searchService = _searchServiceFactory()) { searchResult = await searchService.Value.SearchCompartmentAsync( compartmentType : KnownResourceTypes.Patient, compartmentId : progress.TriggeringResourceId, resourceType : null, queryParametersList, cancellationToken); } await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, cancellationToken); if (searchResult.ContinuationToken == null) { // No more continuation token, we are done. break; } await ProcessProgressChange(exportJobConfiguration, progress, queryParametersList, searchResult.ContinuationToken, false, cancellationToken); currentBatchId = batchIdPrefix + '-' + progress.Page.ToString("d6"); } // Commit one last time for any pending changes. await _exportDestinationClient.CommitAsync(exportJobConfiguration, cancellationToken); await UpdateJobRecordAsync(cancellationToken); }
private async Task RunExportSearch( ExportJobConfiguration exportJobConfiguration, ExportJobProgress progress, List <Tuple <string, string> > sharedQueryParametersList, CancellationToken cancellationToken) { EnsureArg.IsNotNull(exportJobConfiguration, nameof(exportJobConfiguration)); EnsureArg.IsNotNull(progress, nameof(progress)); EnsureArg.IsNotNull(sharedQueryParametersList, nameof(sharedQueryParametersList)); // Current batch will be used to organize a set of search results into a group so that they can be committed together. string currentBatchId = progress.Page.ToString("d6"); List <Tuple <string, string> > queryParametersList = new List <Tuple <string, string> >(sharedQueryParametersList); if (progress.ContinuationToken != null) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken)); } if (_exportJobRecord.ExportType == ExportJobType.Patient) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.Type, KnownResourceTypes.Patient)); } else if (_exportJobRecord.ExportType == ExportJobType.All && !string.IsNullOrEmpty(_exportJobRecord.ResourceType)) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.Type, _exportJobRecord.ResourceType)); } // Process the export if: // 1. There is continuation token, which means there is more resource to be exported. // 2. There is no continuation token but the page is 0, which means it's the initial export. while (progress.ContinuationToken != null || progress.Page == 0) { SearchResult searchResult = null; // Search and process the results. switch (_exportJobRecord.ExportType) { case ExportJobType.All: case ExportJobType.Patient: using (IScoped <ISearchService> searchService = _searchServiceFactory()) { searchResult = await searchService.Value.SearchAsync( resourceType : null, queryParametersList, cancellationToken); } break; case ExportJobType.Group: searchResult = await GetGroupPatients( _exportJobRecord.GroupId, queryParametersList, _exportJobRecord.QueuedTime, cancellationToken); break; } if (_exportJobRecord.ExportType == ExportJobType.Patient || _exportJobRecord.ExportType == ExportJobType.Group) { uint resultIndex = 0; foreach (SearchResultEntry result in searchResult.Results) { // If a job is resumed in the middle of processing patient compartment resources it will skip patients it has already exported compartment information for. // This assumes the order of the search results is the same every time the same search is performed. if (progress.SubSearch != null && result.Resource.ResourceId != progress.SubSearch.TriggeringResourceId) { resultIndex++; continue; } if (progress.SubSearch == null) { progress.NewSubSearch(result.Resource.ResourceId); } await RunExportCompartmentSearch(exportJobConfiguration, progress.SubSearch, sharedQueryParametersList, cancellationToken, currentBatchId + ":" + resultIndex.ToString("d6")); resultIndex++; progress.ClearSubSearch(); } } await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, cancellationToken); if (searchResult.ContinuationToken == null) { // No more continuation token, we are done. break; } await ProcessProgressChange( exportJobConfiguration, progress, queryParametersList, searchResult.ContinuationToken, forceCommit : _exportJobRecord.ExportType == ExportJobType.Patient || _exportJobRecord.ExportType == ExportJobType.Group, cancellationToken); currentBatchId = progress.Page.ToString("d6"); } // Commit one last time for any pending changes. await _exportDestinationClient.CommitAsync(exportJobConfiguration, cancellationToken); }
public AnonymizedExportDataValidationTests(HttpIntegrationTestFixture <StartupForAnonymizedExportTestProvider> fixture) { _testFhirClient = fixture.TestFhirClient; _exportConfiguration = ((IOptions <ExportJobConfiguration>)(fixture.TestFhirServer as InProcTestFhirServer).Server.Services.GetService(typeof(IOptions <ExportJobConfiguration>))).Value; }