public async Task GivenAnExportJobToResume_WhenExecuted_ThenItShouldExportAllRecordsAsExpected() { // We are using the SearchService to throw an exception in order to simulate the export job task // "crashing" while in the middle of the process. var exportJobRecordWithCommitPages = new ExportJobRecord( new Uri("https://localhost/ExportJob/"), "Patient", "hash", since: PartialDateTime.MinValue, storageAccountConnectionHash: string.Empty, storageAccountUri: _exportJobConfiguration.StorageAccountUri, maximumNumberOfResourcesPerQuery: _exportJobConfiguration.MaximumNumberOfResourcesPerQuery, numberOfPagesPerCommit: 2); SetupExportJobRecordAndOperationDataStore(exportJobRecordWithCommitPages); int numberOfCalls = 0; int numberOfSuccessfulPages = 2; _searchService.SearchAsync( Arg.Any <string>(), Arg.Any <IReadOnlyList <Tuple <string, string> > >(), _cancellationToken) .Returns(x => { int count = numberOfCalls; if (count == numberOfSuccessfulPages) { throw new Exception(); } numberOfCalls++; return(CreateSearchResult( new[] { new SearchResultEntry( new ResourceWrapper( count.ToString(CultureInfo.InvariantCulture), "1", "Patient", new RawResource("data", Core.Models.FhirResourceFormat.Json), null, DateTimeOffset.MinValue, false, null, null, null)), }, continuationToken: "ct")); }); await _exportJobTask.ExecuteAsync(_exportJobRecord, _weakETag, _cancellationToken); string exportedIds = _inMemoryDestinationClient.GetExportedData(new Uri(PatientFileName, UriKind.Relative)); Assert.Equal("01", exportedIds); Assert.NotNull(_exportJobRecord.Progress); // We create a new export job task here to simulate the worker picking up the "old" export job record // and resuming the export process. The export destination client contains data that has // been committed up until the "crash". _inMemoryDestinationClient = new InMemoryExportDestinationClient(); var secondExportJobTask = new ExportJobTask( () => _fhirOperationDataStore.CreateMockScope(), Options.Create(_exportJobConfiguration), () => _searchService.CreateMockScope(), _resourceToByteArraySerializer, _inMemoryDestinationClient, NullLogger <ExportJobTask> .Instance); numberOfSuccessfulPages = 5; await secondExportJobTask.ExecuteAsync(_exportJobRecord, _weakETag, _cancellationToken); exportedIds = _inMemoryDestinationClient.GetExportedData(new Uri(PatientFileName, UriKind.Relative)); Assert.Equal("23", exportedIds); }
/// <inheritdoc /> public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETag, CancellationToken cancellationToken) { EnsureArg.IsNotNull(exportJobRecord, nameof(exportJobRecord)); _exportJobRecord = exportJobRecord; _weakETag = weakETag; _fileManager = new ExportFileManager(_exportJobRecord, _exportDestinationClient); var existingFhirRequestContext = _contextAccessor.FhirRequestContext; try { ExportJobConfiguration exportJobConfiguration = _exportJobConfiguration; string connectionHash = string.IsNullOrEmpty(_exportJobConfiguration.StorageAccountConnection) ? string.Empty : Health.Core.Extensions.StringExtensions.ComputeHash(_exportJobConfiguration.StorageAccountConnection); if (string.IsNullOrEmpty(exportJobRecord.StorageAccountUri)) { if (!string.Equals(exportJobRecord.StorageAccountConnectionHash, connectionHash, StringComparison.Ordinal)) { throw new DestinationConnectionException("Storage account connection string was updated during an export job.", HttpStatusCode.BadRequest); } } else { exportJobConfiguration = new ExportJobConfiguration(); exportJobConfiguration.Enabled = _exportJobConfiguration.Enabled; exportJobConfiguration.StorageAccountUri = exportJobRecord.StorageAccountUri; } if (_exportJobRecord.Filters != null && _exportJobRecord.Filters.Count > 0 && string.IsNullOrEmpty(_exportJobRecord.ResourceType)) { throw new BadRequestException(Resources.TypeFilterWithoutTypeIsUnsupported); } // Connect to export destination using appropriate client. await _exportDestinationClient.ConnectAsync(exportJobConfiguration, cancellationToken, _exportJobRecord.StorageAccountContainerName); // Add a request context so that bundle issues can be added by the SearchOptionFactory var fhirRequestContext = new FhirRequestContext( method: "Export", uriString: "$export", baseUriString: "$export", correlationId: _exportJobRecord.Id, requestHeaders: new Dictionary <string, StringValues>(), responseHeaders: new Dictionary <string, StringValues>()); _contextAccessor.FhirRequestContext = fhirRequestContext; // If we are resuming a job, we can detect that by checking the progress info from the job record. // If it is null, then we know we are processing a new job. if (_exportJobRecord.Progress == null) { _exportJobRecord.Progress = new ExportJobProgress(continuationToken: null, page: 0); } // The intial list of query parameters will not have a continutation token. We will add that later if we get one back // from the search result. var queryParametersList = new List <Tuple <string, string> >() { Tuple.Create(KnownQueryParameterNames.Count, _exportJobRecord.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)), Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{_exportJobRecord.QueuedTime.ToString("o", CultureInfo.InvariantCulture)}"), }; if (_exportJobRecord.Since != null) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.LastUpdated, $"ge{_exportJobRecord.Since}")); } ExportJobProgress progress = _exportJobRecord.Progress; await RunExportSearch(exportJobConfiguration, progress, queryParametersList, cancellationToken); await CompleteJobAsync(OperationStatus.Completed, cancellationToken); _logger.LogTrace("Successfully completed the job."); } catch (JobConflictException) { // The export job was updated externally. There might be some additional resources that were exported // but we will not be updating the job record. _logger.LogTrace("The job was updated by another process."); } catch (DestinationConnectionException dce) { _logger.LogError(dce, "Can't connect to destination. The job will be marked as failed."); _exportJobRecord.FailureDetails = new JobFailureDetails(dce.Message, dce.StatusCode); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } catch (ResourceNotFoundException rnfe) { _logger.LogError(rnfe, "Can't find specified resource. The job will be marked as failed."); _exportJobRecord.FailureDetails = new JobFailureDetails(rnfe.Message, HttpStatusCode.BadRequest); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } catch (FailedToParseAnonymizationConfigurationException ex) { _logger.LogError(ex, "Failed to parse anonymization configuration. The job will be marked as failed."); _exportJobRecord.FailureDetails = new JobFailureDetails(ex.Message, HttpStatusCode.BadRequest); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } catch (AnonymizationConfigurationNotFoundException ex) { _logger.LogError(ex, "Cannot found anonymization configuration. The job will be marked as failed."); _exportJobRecord.FailureDetails = new JobFailureDetails(ex.Message, HttpStatusCode.BadRequest); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } catch (AnonymizationConfigurationFetchException ex) { _logger.LogError(ex, "Failed to fetch anonymization configuration file. The job will be marked as failed."); _exportJobRecord.FailureDetails = new JobFailureDetails(ex.Message, HttpStatusCode.BadRequest); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } catch (Exception ex) { // The job has encountered an error it cannot recover from. // Try to update the job to failed state. _logger.LogError(ex, "Encountered an unhandled exception. The job will be marked as failed."); _exportJobRecord.FailureDetails = new JobFailureDetails(Resources.UnknownError, HttpStatusCode.InternalServerError); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } finally { _contextAccessor.FhirRequestContext = existingFhirRequestContext; } }
/// <inheritdoc /> public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETag, CancellationToken cancellationToken) { EnsureArg.IsNotNull(exportJobRecord, nameof(exportJobRecord)); _exportJobRecord = exportJobRecord; _weakETag = weakETag; try { // Connect to export destination using appropriate client. await _exportDestinationClient.ConnectAsync(cancellationToken, _exportJobRecord.Id); // If we are resuming a job, we can detect that by checking the progress info from the job record. // If it is null, then we know we are processing a new job. if (_exportJobRecord.Progress == null) { _exportJobRecord.Progress = new ExportJobProgress(continuationToken: null, page: 0); } ExportJobProgress progress = _exportJobRecord.Progress; // Current batch will be used to organize a set of search results into a group so that they can be committed together. uint currentBatchId = progress.Page; // The intial list of query parameters will not have a continutation token. We will add that later if we get one back // from the search result. var queryParametersList = new List <Tuple <string, string> >() { Tuple.Create(KnownQueryParameterNames.Count, _exportJobConfiguration.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)), Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{_exportJobRecord.QueuedTime.ToString("o", CultureInfo.InvariantCulture)}"), }; if (_exportJobRecord.Since != null) { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.LastUpdated, $"ge{_exportJobRecord.Since}")); } // Process the export if: // 1. There is continuation token, which means there is more resource to be exported. // 2. There is no continuation token but the page is 0, which means it's the initial export. while (progress.ContinuationToken != null || progress.Page == 0) { SearchResult searchResult; // Search and process the results. using (IScoped <ISearchService> searchService = _searchServiceFactory()) { searchResult = await searchService.Value.SearchAsync( _exportJobRecord.ResourceType, queryParametersList, cancellationToken); } await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, cancellationToken); if (searchResult.ContinuationToken == null) { // No more continuation token, we are done. break; } // Update the continuation token in local cache and queryParams. // We will add or udpate the continuation token to the end of the query parameters list. progress.UpdateContinuationToken(searchResult.ContinuationToken); if (queryParametersList[queryParametersList.Count - 1].Item1 == KnownQueryParameterNames.ContinuationToken) { queryParametersList[queryParametersList.Count - 1] = Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken); } else { queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken)); } if (progress.Page % _exportJobConfiguration.NumberOfPagesPerCommit == 0) { // Commit the changes. await _exportDestinationClient.CommitAsync(cancellationToken); // Update the job record. await UpdateJobRecordAsync(cancellationToken); currentBatchId = progress.Page; } } // Commit one last time for any pending changes. await _exportDestinationClient.CommitAsync(cancellationToken); await CompleteJobAsync(OperationStatus.Completed, cancellationToken); _logger.LogTrace("Successfully completed the job."); } catch (JobConflictException) { // The export job was updated externally. There might be some additional resources that were exported // but we will not be updating the job record. _logger.LogTrace("The job was updated by another process."); } catch (DestinationConnectionException dce) { _logger.LogError(dce, "Can't connect to destination. The job will be marked as failed."); _exportJobRecord.FailureDetails = new ExportJobFailureDetails(dce.Message, dce.StatusCode); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } catch (Exception ex) { // The job has encountered an error it cannot recover from. // Try to update the job to failed state. _logger.LogError(ex, "Encountered an unhandled exception. The job will be marked as failed."); _exportJobRecord.FailureDetails = new ExportJobFailureDetails(Resources.UnknownError, HttpStatusCode.InternalServerError); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } }
public Task <ExportJobOutcome> UpdateExportJobAsync(ExportJobRecord jobRecord, WeakETag eTag, CancellationToken cancellationToken) { throw new NotImplementedException(); }
public async Task <CreateExportResponse> Handle(CreateExportRequest request, CancellationToken cancellationToken) { EnsureArg.IsNotNull(request, nameof(request)); if (await _authorizationService.CheckAccess(DataActions.Export) != DataActions.Export) { throw new UnauthorizedFhirActionException(); } IReadOnlyCollection <KeyValuePair <string, string> > requestorClaims = _claimsExtractor.Extract()? .OrderBy(claim => claim.Key, StringComparer.Ordinal).ToList(); // Compute the hash of the job. var hashObject = new { request.RequestUri, RequestorClaims = requestorClaims, }; string hash = JsonConvert.SerializeObject(hashObject).ComputeHash(); string storageAccountConnectionHash = string.IsNullOrEmpty(_exportJobConfiguration.StorageAccountConnection) ? string.Empty : Microsoft.Health.Core.Extensions.StringExtensions.ComputeHash(_exportJobConfiguration.StorageAccountConnection); // Check to see if a matching job exists or not. If a matching job exists, we will return that instead. // Otherwise, we will create a new export job. This will be a best effort since the likelihood of this happen should be small. ExportJobOutcome outcome = await _fhirOperationDataStore.GetExportJobByHashAsync(hash, cancellationToken); ExportJobFormatConfiguration formatConfiguration = null; if (request.FormatName != null) { formatConfiguration = _exportJobConfiguration.Formats?.FirstOrDefault( (ExportJobFormatConfiguration formatConfig) => formatConfig.Name.Equals(request.FormatName, StringComparison.OrdinalIgnoreCase)); if (formatConfiguration == null) { throw new BadRequestException(Resources.ExportFormatNotFound); } } formatConfiguration ??= _exportJobConfiguration.Formats?.FirstOrDefault( (ExportJobFormatConfiguration formatConfig) => formatConfig.Default); formatConfiguration ??= new ExportJobFormatConfiguration() { Format = request.ContainerName == null ? ExportFormatTags.ResourceName : $"{ExportFormatTags.Timestamp}-{ExportFormatTags.Id}/{ExportFormatTags.ResourceName}", }; if (outcome == null) { var jobRecord = new ExportJobRecord( request.RequestUri, request.RequestType, formatConfiguration.Format, request.ResourceType, hash, requestorClaims, request.Since, request.GroupId, storageAccountConnectionHash, _exportJobConfiguration.StorageAccountUri, request.AnonymizationConfigurationLocation, request.AnonymizationConfigurationFileETag, _exportJobConfiguration.MaximumNumberOfResourcesPerQuery, _exportJobConfiguration.NumberOfPagesPerCommit, request.ContainerName); outcome = await _fhirOperationDataStore.CreateExportJobAsync(jobRecord, cancellationToken); } return(new CreateExportResponse(outcome.JobRecord.Id)); }
/// <inheritdoc /> public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETag, CancellationToken cancellationToken) { EnsureArg.IsNotNull(exportJobRecord, nameof(exportJobRecord)); _exportJobRecord = exportJobRecord; _weakETag = weakETag; try { // Get destination type from secret store and connect to the destination using appropriate client. await GetDestinationInfoAndConnectAsync(cancellationToken); // If we are resuming a job, we can detect that by checking the progress info from the job record. // If it is null, then we know we are processing a new job. if (_exportJobRecord.Progress == null) { _exportJobRecord.Progress = new ExportJobProgress(continuationToken: null, page: 0); } ExportJobProgress progress = _exportJobRecord.Progress; // Current batch will be used to organize a set of search results into a group so that they can be committed together. uint currentBatchId = progress.Page; // The first item is placeholder for continuation token so that it can be updated efficiently later. var queryParameters = new Tuple <string, string>[] { Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken), Tuple.Create(KnownQueryParameterNames.Count, _exportJobConfiguration.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)), Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{_exportJobRecord.QueuedTime.ToString("o", CultureInfo.InvariantCulture)}"), }; // Process the export if: // 1. There is continuation token, which means there is more resource to be exported. // 2. There is no continuation token but the page is 0, which means it's the initial export. while (progress.ContinuationToken != null || progress.Page == 0) { SearchResult searchResult; // Search and process the results. using (IScoped <ISearchService> searchService = _searchServiceFactory()) { // If the continuation token is null, then we will exclude it. Calculate the offset and count to be passed in. int offset = queryParameters[0].Item2 == null ? 1 : 0; searchResult = await searchService.Value.SearchAsync( _exportJobRecord.ResourceType, new ArraySegment <Tuple <string, string> >(queryParameters, offset, queryParameters.Length - offset), cancellationToken); } await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, cancellationToken); if (searchResult.ContinuationToken == null) { // No more continuation token, we are done. break; } // Update the continuation token (local cache). progress.UpdateContinuationToken(searchResult.ContinuationToken); queryParameters[0] = Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken); if (progress.Page % _exportJobConfiguration.NumberOfPagesPerCommit == 0) { // Commit the changes. await _exportDestinationClient.CommitAsync(cancellationToken); // Update the job record. await UpdateJobRecordAsync(cancellationToken); currentBatchId = progress.Page; } } // Commit one last time for any pending changes. await _exportDestinationClient.CommitAsync(cancellationToken); await CompleteJobAsync(OperationStatus.Completed, cancellationToken); _logger.LogTrace("Successfully completed the job."); try { // Best effort to delete the secret. If it fails to delete, then move on. await _secretStore.DeleteSecretAsync(_exportJobRecord.SecretName, cancellationToken); } catch (Exception ex) { _logger.LogWarning(ex, "Failed to delete the secret."); } } catch (JobConflictException) { // The export job was updated externally. There might be some additional resources that were exported // but we will not be updating the job record. _logger.LogTrace("The job was updated by another process."); } catch (SecretStoreException sse) { _logger.LogError(sse, "Secret store error. The job will be marked as failed."); _exportJobRecord.FailureDetails = new ExportJobFailureDetails(sse.Message, sse.ResponseStatusCode); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } catch (DestinationConnectionException dce) { _logger.LogError(dce, "Can't connect to destination. The job will be marked as failed."); _exportJobRecord.FailureDetails = new ExportJobFailureDetails(dce.Message, dce.StatusCode); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } catch (Exception ex) { // The job has encountered an error it cannot recover from. // Try to update the job to failed state. _logger.LogError(ex, "Encountered an unhandled exception. The job will be marked as failed."); _exportJobRecord.FailureDetails = new ExportJobFailureDetails(Resources.UnknownError, HttpStatusCode.InternalServerError); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } }
private ExportJobOutcome CreateExportJobOutcome(ExportJobRecord exportJobRecord, WeakETag weakETag = null) { return(new ExportJobOutcome( exportJobRecord, weakETag ?? WeakETag.FromVersionId("123"))); }