private async Task SearchCompartmentWithFilter( ExportJobConfiguration exportJobConfiguration, ExportJobProgress progress, string resourceType, List <Tuple <string, string> > queryParametersList, IAnonymizer anonymizer, string batchIdPrefix, CancellationToken cancellationToken) { // Current batch will be used to organize a set of search results into a group so that they can be committed together. string currentBatchId = batchIdPrefix + "-" + progress.Page.ToString("d6"); // Process the export if: // 1. There is continuation token, which means there is more resource to be exported. // 2. There is no continuation token but the page is 0, which means it's the initial export. while (progress.ContinuationToken != null || progress.Page == 0) { SearchResult searchResult = null; // Search and process the results. using (IScoped <ISearchService> searchService = _searchServiceFactory()) { searchResult = await searchService.Value.SearchCompartmentAsync( compartmentType : KnownResourceTypes.Patient, compartmentId : progress.TriggeringResourceId, resourceType : resourceType, queryParametersList, cancellationToken); } await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, anonymizer, cancellationToken); if (searchResult.ContinuationToken == null) { // No more continuation token, we are done. break; } await ProcessProgressChange(exportJobConfiguration, progress, queryParametersList, searchResult.ContinuationToken, false, cancellationToken); currentBatchId = batchIdPrefix + '-' + progress.Page.ToString("d6"); } // Commit one last time for any pending changes. await _exportDestinationClient.CommitAsync(exportJobConfiguration, cancellationToken); progress.MarkFilterFinished(); await UpdateJobRecordAsync(cancellationToken); }
private async Task ProcessFilterForCompartment( ExportJobConfiguration exportJobConfiguration, ExportJobProgress exportJobProgress, List <Tuple <string, string> > queryParametersList, string batchIdPrefix, CancellationToken cancellationToken) { var index = _exportJobRecord.Filters.IndexOf(exportJobProgress.CurrentFilter); List <Tuple <string, string> > filterQueryParametersList = new List <Tuple <string, string> >(queryParametersList); foreach (var param in exportJobProgress.CurrentFilter.Parameters) { filterQueryParametersList.Add(param); } await SearchCompartmentWithFilter(exportJobConfiguration, exportJobProgress, exportJobProgress.CurrentFilter.ResourceType, filterQueryParametersList, batchIdPrefix + index, cancellationToken); }
private async Task ProcessFilter( ExportJobConfiguration exportJobConfiguration, ExportJobProgress exportJobProgress, List <Tuple <string, string> > queryParametersList, List <Tuple <string, string> > sharedQueryParametersList, IAnonymizer anonymizer, string batchIdPrefix, CancellationToken cancellationToken) { var index = _exportJobRecord.Filters.IndexOf(exportJobProgress.CurrentFilter); List <Tuple <string, string> > filterQueryParametersList = new List <Tuple <string, string> >(queryParametersList); foreach (var param in exportJobProgress.CurrentFilter.Parameters) { filterQueryParametersList.Add(param); } await SearchWithFilter(exportJobConfiguration, exportJobProgress, exportJobProgress.CurrentFilter.ResourceType, filterQueryParametersList, sharedQueryParametersList, anonymizer, batchIdPrefix + index + "-", cancellationToken); exportJobProgress.MarkFilterFinished(); await UpdateJobRecordAsync(cancellationToken); }
private async Task ProcessProgressChange( ExportJobConfiguration exportJobConfiguration, ExportJobProgress progress, List <Tuple <string, string> > queryParametersList, string continuationToken, bool forceCommit, CancellationToken cancellationToken) { // Update the continuation token in local cache and queryParams. // We will add or udpate the continuation token in the query parameters list. progress.UpdateContinuationToken(continuationToken); bool replacedContinuationToken = false; for (int index = 0; index < queryParametersList.Count; index++) { if (queryParametersList[index].Item1 == KnownQueryParameterNames.ContinuationToken) { queryParametersList[index] = Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken); replacedContinuationToken = true; } } if (!replacedContinuationToken) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken)); } if (progress.Page % _exportJobRecord.NumberOfPagesPerCommit == 0 || forceCommit) { // Commit the changes. await _exportDestinationClient.CommitAsync(exportJobConfiguration, cancellationToken); // Update the job record. await UpdateJobRecordAsync(cancellationToken); } }
/// <inheritdoc /> public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETag, CancellationToken cancellationToken) { EnsureArg.IsNotNull(exportJobRecord, nameof(exportJobRecord)); _exportJobRecord = exportJobRecord; _weakETag = weakETag; try { ExportJobConfiguration exportJobConfiguration = _exportJobConfiguration; string connectionHash = string.IsNullOrEmpty(_exportJobConfiguration.StorageAccountConnection) ? string.Empty : Microsoft.Health.Core.Extensions.StringExtensions.ComputeHash(_exportJobConfiguration.StorageAccountConnection); if (string.IsNullOrEmpty(exportJobRecord.StorageAccountUri)) { if (!string.Equals(exportJobRecord.StorageAccountConnectionHash, connectionHash, StringComparison.Ordinal)) { throw new DestinationConnectionException("Storage account connection string was updated during an export job.", HttpStatusCode.BadRequest); } } else { exportJobConfiguration = new ExportJobConfiguration(); exportJobConfiguration.Enabled = _exportJobConfiguration.Enabled; exportJobConfiguration.StorageAccountUri = exportJobRecord.StorageAccountUri; } // Connect to export destination using appropriate client. await _exportDestinationClient.ConnectAsync(exportJobConfiguration, cancellationToken, _exportJobRecord.Id); // If we are resuming a job, we can detect that by checking the progress info from the job record. // If it is null, then we know we are processing a new job. if (_exportJobRecord.Progress == null) { _exportJobRecord.Progress = new ExportJobProgress(continuationToken: null, page: 0); } // The intial list of query parameters will not have a continutation token. We will add that later if we get one back // from the search result. var queryParametersList = new List <Tuple <string, string> >() { Tuple.Create(KnownQueryParameterNames.Count, _exportJobRecord.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)), Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{_exportJobRecord.QueuedTime.ToString("o", CultureInfo.InvariantCulture)}"), }; if (_exportJobRecord.Since != null) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.LastUpdated, $"ge{_exportJobRecord.Since}")); } ExportJobProgress progress = _exportJobRecord.Progress; await RunExportSearch(exportJobConfiguration, progress, queryParametersList, cancellationToken); await CompleteJobAsync(OperationStatus.Completed, cancellationToken); _logger.LogTrace("Successfully completed the job."); } catch (JobConflictException) { // The export job was updated externally. There might be some additional resources that were exported // but we will not be updating the job record. _logger.LogTrace("The job was updated by another process."); } catch (DestinationConnectionException dce) { _logger.LogError(dce, "Can't connect to destination. The job will be marked as failed."); _exportJobRecord.FailureDetails = new JobFailureDetails(dce.Message, dce.StatusCode); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } catch (ResourceNotFoundException rnfe) { _logger.LogError(rnfe, "Can't find specified resource. The job will be marked as failed."); _exportJobRecord.FailureDetails = new JobFailureDetails(rnfe.Message, HttpStatusCode.BadRequest); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } catch (Exception ex) { // The job has encountered an error it cannot recover from. // Try to update the job to failed state. _logger.LogError(ex, "Encountered an unhandled exception. The job will be marked as failed."); _exportJobRecord.FailureDetails = new JobFailureDetails(Resources.UnknownError, HttpStatusCode.InternalServerError); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } }
private async Task RunExportCompartmentSearch( ExportJobConfiguration exportJobConfiguration, ExportJobProgress progress, List <Tuple <string, string> > sharedQueryParametersList, CancellationToken cancellationToken, string batchIdPrefix = "") { EnsureArg.IsNotNull(exportJobConfiguration, nameof(exportJobConfiguration)); EnsureArg.IsNotNull(progress, nameof(progress)); EnsureArg.IsNotNull(sharedQueryParametersList, nameof(sharedQueryParametersList)); // Current batch will be used to organize a set of search results into a group so that they can be committed together. string currentBatchId = batchIdPrefix + "-" + progress.Page.ToString("d6"); List <Tuple <string, string> > queryParametersList = new List <Tuple <string, string> >(sharedQueryParametersList); if (progress.ContinuationToken != null) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken)); } if (!string.IsNullOrEmpty(_exportJobRecord.ResourceType)) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.Type, _exportJobRecord.ResourceType)); } // Process the export if: // 1. There is continuation token, which means there is more resource to be exported. // 2. There is no continuation token but the page is 0, which means it's the initial export. while (progress.ContinuationToken != null || progress.Page == 0) { SearchResult searchResult = null; // Search and process the results. using (IScoped <ISearchService> searchService = _searchServiceFactory()) { searchResult = await searchService.Value.SearchCompartmentAsync( compartmentType : KnownResourceTypes.Patient, compartmentId : progress.TriggeringResourceId, resourceType : null, queryParametersList, cancellationToken); } await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, cancellationToken); if (searchResult.ContinuationToken == null) { // No more continuation token, we are done. break; } await ProcessProgressChange(exportJobConfiguration, progress, queryParametersList, searchResult.ContinuationToken, false, cancellationToken); currentBatchId = batchIdPrefix + '-' + progress.Page.ToString("d6"); } // Commit one last time for any pending changes. await _exportDestinationClient.CommitAsync(exportJobConfiguration, cancellationToken); await UpdateJobRecordAsync(cancellationToken); }
private async Task RunExportSearch( ExportJobConfiguration exportJobConfiguration, ExportJobProgress progress, List <Tuple <string, string> > sharedQueryParametersList, CancellationToken cancellationToken) { EnsureArg.IsNotNull(exportJobConfiguration, nameof(exportJobConfiguration)); EnsureArg.IsNotNull(progress, nameof(progress)); EnsureArg.IsNotNull(sharedQueryParametersList, nameof(sharedQueryParametersList)); // Current batch will be used to organize a set of search results into a group so that they can be committed together. string currentBatchId = progress.Page.ToString("d6"); List <Tuple <string, string> > queryParametersList = new List <Tuple <string, string> >(sharedQueryParametersList); if (progress.ContinuationToken != null) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken)); } if (_exportJobRecord.ExportType == ExportJobType.Patient) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.Type, KnownResourceTypes.Patient)); } else if (_exportJobRecord.ExportType == ExportJobType.All && !string.IsNullOrEmpty(_exportJobRecord.ResourceType)) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.Type, _exportJobRecord.ResourceType)); } // Process the export if: // 1. There is continuation token, which means there is more resource to be exported. // 2. There is no continuation token but the page is 0, which means it's the initial export. while (progress.ContinuationToken != null || progress.Page == 0) { SearchResult searchResult = null; // Search and process the results. switch (_exportJobRecord.ExportType) { case ExportJobType.All: case ExportJobType.Patient: using (IScoped <ISearchService> searchService = _searchServiceFactory()) { searchResult = await searchService.Value.SearchAsync( resourceType : null, queryParametersList, cancellationToken); } break; case ExportJobType.Group: searchResult = await GetGroupPatients( _exportJobRecord.GroupId, queryParametersList, _exportJobRecord.QueuedTime, cancellationToken); break; } if (_exportJobRecord.ExportType == ExportJobType.Patient || _exportJobRecord.ExportType == ExportJobType.Group) { uint resultIndex = 0; foreach (SearchResultEntry result in searchResult.Results) { // If a job is resumed in the middle of processing patient compartment resources it will skip patients it has already exported compartment information for. // This assumes the order of the search results is the same every time the same search is performed. if (progress.SubSearch != null && result.Resource.ResourceId != progress.SubSearch.TriggeringResourceId) { resultIndex++; continue; } if (progress.SubSearch == null) { progress.NewSubSearch(result.Resource.ResourceId); } await RunExportCompartmentSearch(exportJobConfiguration, progress.SubSearch, sharedQueryParametersList, cancellationToken, currentBatchId + ":" + resultIndex.ToString("d6")); resultIndex++; progress.ClearSubSearch(); } } await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, cancellationToken); if (searchResult.ContinuationToken == null) { // No more continuation token, we are done. break; } await ProcessProgressChange( exportJobConfiguration, progress, queryParametersList, searchResult.ContinuationToken, forceCommit : _exportJobRecord.ExportType == ExportJobType.Patient || _exportJobRecord.ExportType == ExportJobType.Group, cancellationToken); currentBatchId = progress.Page.ToString("d6"); } // Commit one last time for any pending changes. await _exportDestinationClient.CommitAsync(exportJobConfiguration, cancellationToken); }
/// <inheritdoc /> public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETag, CancellationToken cancellationToken) { EnsureArg.IsNotNull(exportJobRecord, nameof(exportJobRecord)); _exportJobRecord = exportJobRecord; _weakETag = weakETag; _fileManager = new ExportFileManager(_exportJobRecord, _exportDestinationClient); var existingFhirRequestContext = _contextAccessor.FhirRequestContext; try { ExportJobConfiguration exportJobConfiguration = _exportJobConfiguration; // Add a request context so that bundle issues can be added by the SearchOptionFactory var fhirRequestContext = new FhirRequestContext( method: "Export", uriString: "$export", baseUriString: "$export", correlationId: _exportJobRecord.Id, requestHeaders: new Dictionary <string, StringValues>(), responseHeaders: new Dictionary <string, StringValues>()) { IsBackgroundTask = true, }; _contextAccessor.FhirRequestContext = fhirRequestContext; string connectionHash = string.IsNullOrEmpty(_exportJobConfiguration.StorageAccountConnection) ? string.Empty : Health.Core.Extensions.StringExtensions.ComputeHash(_exportJobConfiguration.StorageAccountConnection); if (string.IsNullOrEmpty(exportJobRecord.StorageAccountUri)) { if (!string.Equals(exportJobRecord.StorageAccountConnectionHash, connectionHash, StringComparison.Ordinal)) { throw new DestinationConnectionException("Storage account connection string was updated during an export job.", HttpStatusCode.BadRequest); } } else { exportJobConfiguration = new ExportJobConfiguration(); exportJobConfiguration.Enabled = _exportJobConfiguration.Enabled; exportJobConfiguration.StorageAccountUri = exportJobRecord.StorageAccountUri; } if (_exportJobRecord.Filters != null && _exportJobRecord.Filters.Count > 0 && string.IsNullOrEmpty(_exportJobRecord.ResourceType)) { throw new BadRequestException(Resources.TypeFilterWithoutTypeIsUnsupported); } // Connect to export destination using appropriate client. await _exportDestinationClient.ConnectAsync(exportJobConfiguration, cancellationToken, _exportJobRecord.StorageAccountContainerName); // If we are resuming a job, we can detect that by checking the progress info from the job record. // If it is null, then we know we are processing a new job. if (_exportJobRecord.Progress == null) { _exportJobRecord.Progress = new ExportJobProgress(continuationToken: null, page: 0); } // The intial list of query parameters will not have a continutation token. We will add that later if we get one back // from the search result. var queryParametersList = new List <Tuple <string, string> >() { Tuple.Create(KnownQueryParameterNames.Count, _exportJobRecord.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)), Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{_exportJobRecord.QueuedTime.ToString("o", CultureInfo.InvariantCulture)}"), }; if (_exportJobRecord.Since != null) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.LastUpdated, $"ge{_exportJobRecord.Since}")); } ExportJobProgress progress = _exportJobRecord.Progress; await RunExportSearch(exportJobConfiguration, progress, queryParametersList, cancellationToken); await CompleteJobAsync(OperationStatus.Completed, cancellationToken); _logger.LogTrace("Successfully completed the job."); } catch (JobConflictException) { // The export job was updated externally. There might be some additional resources that were exported // but we will not be updating the job record. _logger.LogTrace("The job was updated by another process."); } catch (DestinationConnectionException dce) { _logger.LogError(dce, "Can't connect to destination. The job will be marked as failed."); _exportJobRecord.FailureDetails = new JobFailureDetails(dce.Message, dce.StatusCode); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } catch (ResourceNotFoundException rnfe) { _logger.LogError(rnfe, "Can't find specified resource. The job will be marked as failed."); _exportJobRecord.FailureDetails = new JobFailureDetails(rnfe.Message, HttpStatusCode.BadRequest); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } catch (FailedToParseAnonymizationConfigurationException ex) { _logger.LogError(ex, "Failed to parse anonymization configuration. The job will be marked as failed."); _exportJobRecord.FailureDetails = new JobFailureDetails(ex.Message, HttpStatusCode.BadRequest); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } catch (AnonymizationConfigurationNotFoundException ex) { _logger.LogError(ex, "Cannot found anonymization configuration. The job will be marked as failed."); _exportJobRecord.FailureDetails = new JobFailureDetails(ex.Message, HttpStatusCode.BadRequest); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } catch (AnonymizationConfigurationFetchException ex) { _logger.LogError(ex, "Failed to fetch anonymization configuration file. The job will be marked as failed."); _exportJobRecord.FailureDetails = new JobFailureDetails(ex.Message, HttpStatusCode.BadRequest); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } catch (Exception ex) { // The job has encountered an error it cannot recover from. // Try to update the job to failed state. _logger.LogError(ex, "Encountered an unhandled exception. The job will be marked as failed."); _exportJobRecord.FailureDetails = new JobFailureDetails(Resources.UnknownError, HttpStatusCode.InternalServerError); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } finally { _contextAccessor.FhirRequestContext = existingFhirRequestContext; } }
private async Task RunExportCompartmentSearch( ExportJobConfiguration exportJobConfiguration, ExportJobProgress progress, List <Tuple <string, string> > sharedQueryParametersList, IAnonymizer anonymizer, CancellationToken cancellationToken, string batchIdPrefix = "") { EnsureArg.IsNotNull(exportJobConfiguration, nameof(exportJobConfiguration)); EnsureArg.IsNotNull(progress, nameof(progress)); EnsureArg.IsNotNull(sharedQueryParametersList, nameof(sharedQueryParametersList)); List <Tuple <string, string> > queryParametersList = new List <Tuple <string, string> >(sharedQueryParametersList); if (progress.ContinuationToken != null) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken)); } var requestedResourceTypes = _exportJobRecord.ResourceType?.Split(','); var filteredResources = new HashSet <string>(StringComparer.OrdinalIgnoreCase); if (_exportJobRecord.Filters != null) { foreach (var filter in _exportJobRecord.Filters) { filteredResources.Add(filter.ResourceType); } } if (progress.CurrentFilter != null) { await ProcessFilterForCompartment(exportJobConfiguration, progress, queryParametersList, anonymizer, batchIdPrefix + "-filter", cancellationToken); } if (_exportJobRecord.Filters != null) { foreach (var filter in _exportJobRecord.Filters) { if (!progress.CompletedFilters.Contains(filter) && requestedResourceTypes != null && requestedResourceTypes.Contains(filter.ResourceType, StringComparison.OrdinalIgnoreCase)) { progress.SetFilter(filter); await ProcessFilterForCompartment(exportJobConfiguration, progress, queryParametersList, anonymizer, batchIdPrefix + "-filter", cancellationToken); } } } if (_exportJobRecord.Filters == null || _exportJobRecord.Filters.Count == 0 || !requestedResourceTypes.All(resourceType => filteredResources.Contains(resourceType))) { if (requestedResourceTypes != null) { List <string> resources = new List <string>(); foreach (var resource in requestedResourceTypes) { if (!filteredResources.Contains(resource)) { resources.Add(resource); } } if (resources.Count > 0) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.Type, resources.JoinByOrSeparator())); } } await SearchCompartmentWithFilter(exportJobConfiguration, progress, null, queryParametersList, anonymizer, batchIdPrefix, cancellationToken); } }
private async Task SearchWithFilter( ExportJobConfiguration exportJobConfiguration, ExportJobProgress progress, string resourceType, List <Tuple <string, string> > queryParametersList, List <Tuple <string, string> > sharedQueryParametersList, IAnonymizer anonymizer, string batchIdPrefix, CancellationToken cancellationToken) { // Current batch will be used to organize a set of search results into a group so that they can be committed together. string currentBatchId = batchIdPrefix + progress.Page.ToString("d6"); // Process the export if: // 1. There is continuation token, which means there is more resource to be exported. // 2. There is no continuation token but the page is 0, which means it's the initial export. while (progress.ContinuationToken != null || progress.Page == 0) { SearchResult searchResult = null; // Search and process the results. switch (_exportJobRecord.ExportType) { case ExportJobType.All: case ExportJobType.Patient: using (IScoped <ISearchService> searchService = _searchServiceFactory()) { searchResult = await searchService.Value.SearchAsync( resourceType : resourceType, queryParametersList, cancellationToken); } break; case ExportJobType.Group: searchResult = await GetGroupPatients( _exportJobRecord.GroupId, queryParametersList, _exportJobRecord.QueuedTime, cancellationToken); break; } if (_exportJobRecord.ExportType == ExportJobType.Patient || _exportJobRecord.ExportType == ExportJobType.Group) { uint resultIndex = 0; foreach (SearchResultEntry result in searchResult.Results) { // If a job is resumed in the middle of processing patient compartment resources it will skip patients it has already exported compartment information for. // This assumes the order of the search results is the same every time the same search is performed. if (progress.SubSearch != null && result.Resource.ResourceId != progress.SubSearch.TriggeringResourceId) { resultIndex++; continue; } if (progress.SubSearch == null) { progress.NewSubSearch(result.Resource.ResourceId); } await RunExportCompartmentSearch(exportJobConfiguration, progress.SubSearch, sharedQueryParametersList, anonymizer, cancellationToken, currentBatchId + ":" + resultIndex.ToString("d6")); resultIndex++; progress.ClearSubSearch(); } } // Skips processing top level search results if the job only requested resources from the compartments of patients, but didn't want the patients. if (_exportJobRecord.ExportType == ExportJobType.All || string.IsNullOrWhiteSpace(_exportJobRecord.ResourceType) || _exportJobRecord.ResourceType.Contains(KnownResourceTypes.Patient, StringComparison.OrdinalIgnoreCase)) { await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, anonymizer, cancellationToken); } if (searchResult.ContinuationToken == null) { // No more continuation token, we are done. break; } await ProcessProgressChange( exportJobConfiguration, progress, queryParametersList, searchResult.ContinuationToken, forceCommit : _exportJobRecord.ExportType == ExportJobType.Patient || _exportJobRecord.ExportType == ExportJobType.Group, cancellationToken); currentBatchId = batchIdPrefix + progress.Page.ToString("d6"); } // Commit one last time for any pending changes. await _exportDestinationClient.CommitAsync(exportJobConfiguration, cancellationToken); }
private async Task RunExportSearch( ExportJobConfiguration exportJobConfiguration, ExportJobProgress progress, List <Tuple <string, string> > sharedQueryParametersList, CancellationToken cancellationToken) { EnsureArg.IsNotNull(exportJobConfiguration, nameof(exportJobConfiguration)); EnsureArg.IsNotNull(progress, nameof(progress)); EnsureArg.IsNotNull(sharedQueryParametersList, nameof(sharedQueryParametersList)); List <Tuple <string, string> > queryParametersList = new List <Tuple <string, string> >(sharedQueryParametersList); if (progress.ContinuationToken != null) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken)); } var requestedResourceTypes = _exportJobRecord.ResourceType?.Split(','); var filteredResources = new HashSet <string>(StringComparer.OrdinalIgnoreCase); if (_exportJobRecord.Filters != null) { foreach (var filter in _exportJobRecord.Filters) { filteredResources.Add(filter.ResourceType); } } IAnonymizer anonymizer = IsAnonymizedExportJob() ? await CreateAnonymizerAsync(cancellationToken) : null; if (progress.CurrentFilter != null) { await ProcessFilter(exportJobConfiguration, progress, queryParametersList, sharedQueryParametersList, anonymizer, "filter", cancellationToken); } if (_exportJobRecord.Filters != null && _exportJobRecord.Filters.Any(filter => !progress.CompletedFilters.Contains(filter))) { foreach (var filter in _exportJobRecord.Filters) { if (!progress.CompletedFilters.Contains(filter) && requestedResourceTypes != null && requestedResourceTypes.Contains(filter.ResourceType, StringComparison.OrdinalIgnoreCase) && (_exportJobRecord.ExportType == ExportJobType.All || filter.ResourceType.Equals(KnownResourceTypes.Patient, StringComparison.OrdinalIgnoreCase))) { progress.SetFilter(filter); await ProcessFilter(exportJobConfiguration, progress, queryParametersList, sharedQueryParametersList, anonymizer, "filter", cancellationToken); } } } // The unfiltered search should be run if there were no filters specified, there were types requested that didn't have filters for them, or if a Patient/Group level export didn't have filters for Patients. // Examples: // If a patient/group export job with type and type filters is run, but patients aren't in the types requested, the search should be run here but no patients printed to the output // If a patient/group export job with type and type filters is run, and patients are in the types requested and filtered, the search should not be run as patients were searched above // If an export job with type and type filters is run, the search should not be run if all the types were searched above. if (_exportJobRecord.Filters == null || _exportJobRecord.Filters.Count == 0 || (_exportJobRecord.ExportType == ExportJobType.All && !requestedResourceTypes.All(resourceType => filteredResources.Contains(resourceType))) || ((_exportJobRecord.ExportType == ExportJobType.Patient || _exportJobRecord.ExportType == ExportJobType.Group) && !filteredResources.Contains(KnownResourceTypes.Patient))) { if (_exportJobRecord.ExportType == ExportJobType.Patient) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.Type, KnownResourceTypes.Patient)); } else if (_exportJobRecord.ExportType == ExportJobType.All && requestedResourceTypes != null) { List <string> resources = new List <string>(); foreach (var resource in requestedResourceTypes) { if (!filteredResources.Contains(resource)) { resources.Add(resource); } } if (resources.Count > 0) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.Type, resources.JoinByOrSeparator())); } } await SearchWithFilter(exportJobConfiguration, progress, null, queryParametersList, sharedQueryParametersList, anonymizer, string.Empty, cancellationToken); } }
/// <inheritdoc /> public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETag, CancellationToken cancellationToken) { EnsureArg.IsNotNull(exportJobRecord, nameof(exportJobRecord)); _exportJobRecord = exportJobRecord; _weakETag = weakETag; try { ExportJobConfiguration exportJobConfiguration = _exportJobConfiguration; string connectionHash = string.IsNullOrEmpty(_exportJobConfiguration.StorageAccountConnection) ? string.Empty : Microsoft.Health.Core.Extensions.StringExtensions.ComputeHash(_exportJobConfiguration.StorageAccountConnection); if (string.IsNullOrEmpty(exportJobRecord.StorageAccountUri)) { if (!string.Equals(exportJobRecord.StorageAccountConnectionHash, connectionHash, StringComparison.Ordinal)) { throw new DestinationConnectionException("Storage account connection string was updated during an export job.", HttpStatusCode.BadRequest); } } else { exportJobConfiguration = new ExportJobConfiguration(); exportJobConfiguration.Enabled = _exportJobConfiguration.Enabled; exportJobConfiguration.StorageAccountUri = exportJobRecord.StorageAccountUri; } // Connect to export destination using appropriate client. await _exportDestinationClient.ConnectAsync(exportJobConfiguration, cancellationToken, _exportJobRecord.Id); // If we are resuming a job, we can detect that by checking the progress info from the job record. // If it is null, then we know we are processing a new job. if (_exportJobRecord.Progress == null) { _exportJobRecord.Progress = new ExportJobProgress(continuationToken: null, page: 0); } ExportJobProgress progress = _exportJobRecord.Progress; // Current batch will be used to organize a set of search results into a group so that they can be committed together. uint currentBatchId = progress.Page; // The intial list of query parameters will not have a continutation token. We will add that later if we get one back // from the search result. var queryParametersList = new List <Tuple <string, string> >() { Tuple.Create(KnownQueryParameterNames.Count, _exportJobConfiguration.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)), Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{_exportJobRecord.QueuedTime.ToString("o", CultureInfo.InvariantCulture)}"), }; if (_exportJobRecord.Since != null) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.LastUpdated, $"ge{_exportJobRecord.Since}")); } // Process the export if: // 1. There is continuation token, which means there is more resource to be exported. // 2. There is no continuation token but the page is 0, which means it's the initial export. while (progress.ContinuationToken != null || progress.Page == 0) { SearchResult searchResult; // Search and process the results. using (IScoped <ISearchService> searchService = _searchServiceFactory()) { searchResult = await searchService.Value.SearchAsync( _exportJobRecord.ResourceType, queryParametersList, cancellationToken); } await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, cancellationToken); if (searchResult.ContinuationToken == null) { // No more continuation token, we are done. break; } // Update the continuation token in local cache and queryParams. // We will add or udpate the continuation token to the end of the query parameters list. progress.UpdateContinuationToken(searchResult.ContinuationToken); if (queryParametersList[queryParametersList.Count - 1].Item1 == KnownQueryParameterNames.ContinuationToken) { queryParametersList[queryParametersList.Count - 1] = Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken); } else { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken)); } if (progress.Page % _exportJobConfiguration.NumberOfPagesPerCommit == 0) { // Commit the changes. await _exportDestinationClient.CommitAsync(exportJobConfiguration, cancellationToken); // Update the job record. await UpdateJobRecordAsync(cancellationToken); currentBatchId = progress.Page; } } // Commit one last time for any pending changes. await _exportDestinationClient.CommitAsync(exportJobConfiguration, cancellationToken); await CompleteJobAsync(OperationStatus.Completed, cancellationToken); _logger.LogTrace("Successfully completed the job."); } catch (JobConflictException) { // The export job was updated externally. There might be some additional resources that were exported // but we will not be updating the job record. _logger.LogTrace("The job was updated by another process."); } catch (DestinationConnectionException dce) { _logger.LogError(dce, "Can't connect to destination. The job will be marked as failed."); _exportJobRecord.FailureDetails = new JobFailureDetails(dce.Message, dce.StatusCode); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } catch (Exception ex) { // The job has encountered an error it cannot recover from. // Try to update the job to failed state. _logger.LogError(ex, "Encountered an unhandled exception. The job will be marked as failed."); _exportJobRecord.FailureDetails = new JobFailureDetails(Resources.UnknownError, HttpStatusCode.InternalServerError); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } }
/// <inheritdoc /> public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETag, CancellationToken cancellationToken) { EnsureArg.IsNotNull(exportJobRecord, nameof(exportJobRecord)); _exportJobRecord = exportJobRecord; _weakETag = weakETag; try { // Get destination type from secret store. DestinationInfo destinationInfo = await GetDestinationInfo(cancellationToken); // Connect to the destination using appropriate client. _exportDestinationClient = _exportDestinationClientFactory.Create(destinationInfo.DestinationType); await _exportDestinationClient.ConnectAsync(destinationInfo.DestinationConnectionString, cancellationToken, _exportJobRecord.Id); // TODO: For now, always restart from the beginning. We will support resume in another work item. _exportJobRecord.Progress = new ExportJobProgress(continuationToken: null, page: 0); ExportJobProgress progress = _exportJobRecord.Progress; // Current page will be used to organize a set of search results into a group so that they can be committed together. uint currentBatchId = progress.Page; // The first item is placeholder for continuation token so that it can be updated efficiently later. var queryParameters = new Tuple <string, string>[] { null, Tuple.Create(KnownQueryParameterNames.Count, _exportJobConfiguration.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)), Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{_exportJobRecord.QueuedTime.ToString("o", CultureInfo.InvariantCulture)}"), }; // Process the export if: // 1. There is continuation token, which means there is more resource to be exported. // 2. There is no continuation token but the page is 0, which means it's the initial export. while (progress.ContinuationToken != null || progress.Page == 0) { // Commit the changes if necessary. if (progress.Page != 0 && progress.Page % _exportJobConfiguration.NumberOfPagesPerCommit == 0) { await _exportDestinationClient.CommitAsync(cancellationToken); // Update the job record. await UpdateJobRecord(_exportJobRecord, cancellationToken); currentBatchId = progress.Page; } // Set the continuation token. queryParameters[0] = Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken); SearchResult searchResult = await _searchService.SearchAsync(_exportJobRecord.ResourceType, queryParameters, cancellationToken); foreach (ResourceWrapper resourceWrapper in searchResult.Results) { await ProcessResourceWrapperAsync(resourceWrapper, currentBatchId, cancellationToken); } if (searchResult.ContinuationToken == null) { // No more continuation token, we are done. break; } // Update the job record. progress.UpdateContinuationToken(searchResult.ContinuationToken); } // Commit one last time for any pending changes. await _exportDestinationClient.CommitAsync(cancellationToken); _exportJobRecord.Output.AddRange(_resourceTypeToFileInfoMapping.Values); _logger.LogTrace("Successfully completed the job."); await UpdateJobStatus(OperationStatus.Completed, updateEndTimestamp : true, cancellationToken); try { // Best effort to delete the secret. If it fails to delete, then move on. await _secretStore.DeleteSecretAsync(_exportJobRecord.SecretName, cancellationToken); } catch (Exception ex) { _logger.LogWarning(ex, "Failed to delete the secret."); } } catch (JobConflictException) { // The job was updated by another process. _logger.LogWarning("The job was updated by another process."); // TODO: We will want to get the latest and merge the results without updating the status. return; } catch (Exception ex) { // The job has encountered an error it cannot recover from. // Try to update the job to failed state. _logger.LogError(ex, "Encountered an unhandled exception. The job will be marked as failed."); await UpdateJobStatus(OperationStatus.Failed, updateEndTimestamp : true, cancellationToken); } }
/// <inheritdoc /> public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETag, CancellationToken cancellationToken) { EnsureArg.IsNotNull(exportJobRecord, nameof(exportJobRecord)); _exportJobRecord = exportJobRecord; _weakETag = weakETag; try { // Get destination type from secret store and connect to the destination using appropriate client. await GetDestinationInfoAndConnectAsync(cancellationToken); // If we are resuming a job, we can detect that by checking the progress info from the job record. // If it is null, then we know we are processing a new job. if (_exportJobRecord.Progress == null) { _exportJobRecord.Progress = new ExportJobProgress(continuationToken: null, page: 0); } ExportJobProgress progress = _exportJobRecord.Progress; // Current batch will be used to organize a set of search results into a group so that they can be committed together. uint currentBatchId = progress.Page; // The first item is placeholder for continuation token so that it can be updated efficiently later. var queryParameters = new Tuple <string, string>[] { Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken), Tuple.Create(KnownQueryParameterNames.Count, _exportJobConfiguration.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)), Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{_exportJobRecord.QueuedTime.ToString("o", CultureInfo.InvariantCulture)}"), }; // Process the export if: // 1. There is continuation token, which means there is more resource to be exported. // 2. There is no continuation token but the page is 0, which means it's the initial export. while (progress.ContinuationToken != null || progress.Page == 0) { SearchResult searchResult; // Search and process the results. using (IScoped <ISearchService> searchService = _searchServiceFactory()) { // If the continuation token is null, then we will exclude it. Calculate the offset and count to be passed in. int offset = queryParameters[0].Item2 == null ? 1 : 0; searchResult = await searchService.Value.SearchAsync( _exportJobRecord.ResourceType, new ArraySegment <Tuple <string, string> >(queryParameters, offset, queryParameters.Length - offset), cancellationToken); } await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, cancellationToken); if (searchResult.ContinuationToken == null) { // No more continuation token, we are done. break; } // Update the continuation token (local cache). progress.UpdateContinuationToken(searchResult.ContinuationToken); queryParameters[0] = Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken); if (progress.Page % _exportJobConfiguration.NumberOfPagesPerCommit == 0) { // Commit the changes. await _exportDestinationClient.CommitAsync(cancellationToken); // Update the job record. await UpdateJobRecordAsync(cancellationToken); currentBatchId = progress.Page; } } // Commit one last time for any pending changes. await _exportDestinationClient.CommitAsync(cancellationToken); await CompleteJobAsync(OperationStatus.Completed, cancellationToken); _logger.LogTrace("Successfully completed the job."); try { // Best effort to delete the secret. If it fails to delete, then move on. await _secretStore.DeleteSecretAsync(_exportJobRecord.SecretName, cancellationToken); } catch (Exception ex) { _logger.LogWarning(ex, "Failed to delete the secret."); } } catch (JobConflictException) { // The export job was updated externally. There might be some additional resources that were exported // but we will not be updating the job record. _logger.LogTrace("The job was updated by another process."); } catch (Exception ex) { // The job has encountered an error it cannot recover from. // Try to update the job to failed state. _logger.LogError(ex, "Encountered an unhandled exception. The job will be marked as failed."); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } }