public async Task GivenAnExportJobToResume_WhenExecuted_ThenItShouldExportAllRecordsAsExpected()
{
// We are using the SearchService to throw an exception in order to simulate the export job task
// "crashing" while in the middle of the process.
var exportJobRecordWithCommitPages = new ExportJobRecord(
new Uri("https://localhost/ExportJob/"),
"Patient",
"hash",
since: PartialDateTime.MinValue,
storageAccountConnectionHash: string.Empty,
storageAccountUri: _exportJobConfiguration.StorageAccountUri,
maximumNumberOfResourcesPerQuery: _exportJobConfiguration.MaximumNumberOfResourcesPerQuery,
numberOfPagesPerCommit: 2);
SetupExportJobRecordAndOperationDataStore(exportJobRecordWithCommitPages);
int numberOfCalls = 0;
int numberOfSuccessfulPages = 2;
_searchService.SearchAsync(
Arg.Any <string>(),
Arg.Any <IReadOnlyList <Tuple <string, string> > >(),
_cancellationToken)
.Returns(x =>
{
int count = numberOfCalls;
if (count == numberOfSuccessfulPages)
{
throw new Exception();
}
numberOfCalls++;
return(CreateSearchResult(
new[]
{
new SearchResultEntry(
new ResourceWrapper(
count.ToString(CultureInfo.InvariantCulture),
"1",
"Patient",
new RawResource("data", Core.Models.FhirResourceFormat.Json),
null,
DateTimeOffset.MinValue,
false,
null,
null,
null)),
},
continuationToken: "ct"));
});
await _exportJobTask.ExecuteAsync(_exportJobRecord, _weakETag, _cancellationToken);
string exportedIds = _inMemoryDestinationClient.GetExportedData(new Uri(PatientFileName, UriKind.Relative));
Assert.Equal("01", exportedIds);
Assert.NotNull(_exportJobRecord.Progress);
// We create a new export job task here to simulate the worker picking up the "old" export job record
// and resuming the export process. The export destination client contains data that has
// been committed up until the "crash".
_inMemoryDestinationClient = new InMemoryExportDestinationClient();
var secondExportJobTask = new ExportJobTask(
() => _fhirOperationDataStore.CreateMockScope(),
Options.Create(_exportJobConfiguration),
() => _searchService.CreateMockScope(),
_resourceToByteArraySerializer,
_inMemoryDestinationClient,
NullLogger <ExportJobTask> .Instance);
numberOfSuccessfulPages = 5;
await secondExportJobTask.ExecuteAsync(_exportJobRecord, _weakETag, _cancellationToken);
exportedIds = _inMemoryDestinationClient.GetExportedData(new Uri(PatientFileName, UriKind.Relative));
Assert.Equal("23", exportedIds);
}
/// <inheritdoc />
public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETag, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(exportJobRecord, nameof(exportJobRecord));
_exportJobRecord = exportJobRecord;
_weakETag = weakETag;
_fileManager = new ExportFileManager(_exportJobRecord, _exportDestinationClient);
var existingFhirRequestContext = _contextAccessor.FhirRequestContext;
try
{
ExportJobConfiguration exportJobConfiguration = _exportJobConfiguration;
string connectionHash = string.IsNullOrEmpty(_exportJobConfiguration.StorageAccountConnection) ?
string.Empty :
Health.Core.Extensions.StringExtensions.ComputeHash(_exportJobConfiguration.StorageAccountConnection);
if (string.IsNullOrEmpty(exportJobRecord.StorageAccountUri))
{
if (!string.Equals(exportJobRecord.StorageAccountConnectionHash, connectionHash, StringComparison.Ordinal))
{
throw new DestinationConnectionException("Storage account connection string was updated during an export job.", HttpStatusCode.BadRequest);
}
}
else
{
exportJobConfiguration = new ExportJobConfiguration();
exportJobConfiguration.Enabled = _exportJobConfiguration.Enabled;
exportJobConfiguration.StorageAccountUri = exportJobRecord.StorageAccountUri;
}
if (_exportJobRecord.Filters != null &&
_exportJobRecord.Filters.Count > 0 &&
string.IsNullOrEmpty(_exportJobRecord.ResourceType))
{
throw new BadRequestException(Resources.TypeFilterWithoutTypeIsUnsupported);
}
// Connect to export destination using appropriate client.
await _exportDestinationClient.ConnectAsync(exportJobConfiguration, cancellationToken, _exportJobRecord.StorageAccountContainerName);
// Add a request context so that bundle issues can be added by the SearchOptionFactory
var fhirRequestContext = new FhirRequestContext(
method: "Export",
uriString: "$export",
baseUriString: "$export",
correlationId: _exportJobRecord.Id,
requestHeaders: new Dictionary <string, StringValues>(),
responseHeaders: new Dictionary <string, StringValues>());
_contextAccessor.FhirRequestContext = fhirRequestContext;
// If we are resuming a job, we can detect that by checking the progress info from the job record.
// If it is null, then we know we are processing a new job.
if (_exportJobRecord.Progress == null)
{
_exportJobRecord.Progress = new ExportJobProgress(continuationToken: null, page: 0);
}
// The intial list of query parameters will not have a continutation token. We will add that later if we get one back
// from the search result.
var queryParametersList = new List <Tuple <string, string> >()
{
Tuple.Create(KnownQueryParameterNames.Count, _exportJobRecord.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)),
Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{_exportJobRecord.QueuedTime.ToString("o", CultureInfo.InvariantCulture)}"),
};
if (_exportJobRecord.Since != null)
{
queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.LastUpdated, $"ge{_exportJobRecord.Since}"));
}
ExportJobProgress progress = _exportJobRecord.Progress;
await RunExportSearch(exportJobConfiguration, progress, queryParametersList, cancellationToken);
await CompleteJobAsync(OperationStatus.Completed, cancellationToken);
_logger.LogTrace("Successfully completed the job.");
}
catch (JobConflictException)
{
// The export job was updated externally. There might be some additional resources that were exported
// but we will not be updating the job record.
_logger.LogTrace("The job was updated by another process.");
}
catch (DestinationConnectionException dce)
{
_logger.LogError(dce, "Can't connect to destination. The job will be marked as failed.");
_exportJobRecord.FailureDetails = new JobFailureDetails(dce.Message, dce.StatusCode);
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
catch (ResourceNotFoundException rnfe)
{
_logger.LogError(rnfe, "Can't find specified resource. The job will be marked as failed.");
_exportJobRecord.FailureDetails = new JobFailureDetails(rnfe.Message, HttpStatusCode.BadRequest);
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
catch (FailedToParseAnonymizationConfigurationException ex)
{
_logger.LogError(ex, "Failed to parse anonymization configuration. The job will be marked as failed.");
_exportJobRecord.FailureDetails = new JobFailureDetails(ex.Message, HttpStatusCode.BadRequest);
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
catch (AnonymizationConfigurationNotFoundException ex)
{
_logger.LogError(ex, "Cannot found anonymization configuration. The job will be marked as failed.");
_exportJobRecord.FailureDetails = new JobFailureDetails(ex.Message, HttpStatusCode.BadRequest);
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
catch (AnonymizationConfigurationFetchException ex)
{
_logger.LogError(ex, "Failed to fetch anonymization configuration file. The job will be marked as failed.");
_exportJobRecord.FailureDetails = new JobFailureDetails(ex.Message, HttpStatusCode.BadRequest);
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
catch (Exception ex)
{
// The job has encountered an error it cannot recover from.
// Try to update the job to failed state.
_logger.LogError(ex, "Encountered an unhandled exception. The job will be marked as failed.");
_exportJobRecord.FailureDetails = new JobFailureDetails(Resources.UnknownError, HttpStatusCode.InternalServerError);
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
finally
{
_contextAccessor.FhirRequestContext = existingFhirRequestContext;
}
}
/// <inheritdoc />
public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETag, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(exportJobRecord, nameof(exportJobRecord));
_exportJobRecord = exportJobRecord;
_weakETag = weakETag;
try
{
// Connect to export destination using appropriate client.
await _exportDestinationClient.ConnectAsync(cancellationToken, _exportJobRecord.Id);
// If we are resuming a job, we can detect that by checking the progress info from the job record.
// If it is null, then we know we are processing a new job.
if (_exportJobRecord.Progress == null)
{
_exportJobRecord.Progress = new ExportJobProgress(continuationToken: null, page: 0);
}
ExportJobProgress progress = _exportJobRecord.Progress;
// Current batch will be used to organize a set of search results into a group so that they can be committed together.
uint currentBatchId = progress.Page;
// The intial list of query parameters will not have a continutation token. We will add that later if we get one back
// from the search result.
var queryParametersList = new List <Tuple <string, string> >()
{
Tuple.Create(KnownQueryParameterNames.Count, _exportJobConfiguration.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)),
Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{_exportJobRecord.QueuedTime.ToString("o", CultureInfo.InvariantCulture)}"),
};
if (_exportJobRecord.Since != null)
{
queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.LastUpdated, $"ge{_exportJobRecord.Since}"));
}
// Process the export if:
// 1. There is continuation token, which means there is more resource to be exported.
// 2. There is no continuation token but the page is 0, which means it's the initial export.
while (progress.ContinuationToken != null || progress.Page == 0)
{
SearchResult searchResult;
// Search and process the results.
using (IScoped <ISearchService> searchService = _searchServiceFactory())
{
searchResult = await searchService.Value.SearchAsync(
_exportJobRecord.ResourceType,
queryParametersList,
cancellationToken);
}
await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, cancellationToken);
if (searchResult.ContinuationToken == null)
{
// No more continuation token, we are done.
break;
}
// Update the continuation token in local cache and queryParams.
// We will add or udpate the continuation token to the end of the query parameters list.
progress.UpdateContinuationToken(searchResult.ContinuationToken);
if (queryParametersList[queryParametersList.Count - 1].Item1 == KnownQueryParameterNames.ContinuationToken)
{
queryParametersList[queryParametersList.Count - 1] = Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken);
}
else
{
queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken));
}
if (progress.Page % _exportJobConfiguration.NumberOfPagesPerCommit == 0)
{
// Commit the changes.
await _exportDestinationClient.CommitAsync(cancellationToken);
// Update the job record.
await UpdateJobRecordAsync(cancellationToken);
currentBatchId = progress.Page;
}
}
// Commit one last time for any pending changes.
await _exportDestinationClient.CommitAsync(cancellationToken);
await CompleteJobAsync(OperationStatus.Completed, cancellationToken);
_logger.LogTrace("Successfully completed the job.");
}
catch (JobConflictException)
{
// The export job was updated externally. There might be some additional resources that were exported
// but we will not be updating the job record.
_logger.LogTrace("The job was updated by another process.");
}
catch (DestinationConnectionException dce)
{
_logger.LogError(dce, "Can't connect to destination. The job will be marked as failed.");
_exportJobRecord.FailureDetails = new ExportJobFailureDetails(dce.Message, dce.StatusCode);
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
catch (Exception ex)
{
// The job has encountered an error it cannot recover from.
// Try to update the job to failed state.
_logger.LogError(ex, "Encountered an unhandled exception. The job will be marked as failed.");
_exportJobRecord.FailureDetails = new ExportJobFailureDetails(Resources.UnknownError, HttpStatusCode.InternalServerError);
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
}
public Task <ExportJobOutcome> UpdateExportJobAsync(ExportJobRecord jobRecord, WeakETag eTag, CancellationToken cancellationToken)
{
throw new NotImplementedException();
}
public async Task <CreateExportResponse> Handle(CreateExportRequest request, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(request, nameof(request));
if (await _authorizationService.CheckAccess(DataActions.Export) != DataActions.Export)
{
throw new UnauthorizedFhirActionException();
}
IReadOnlyCollection <KeyValuePair <string, string> > requestorClaims = _claimsExtractor.Extract()?
.OrderBy(claim => claim.Key, StringComparer.Ordinal).ToList();
// Compute the hash of the job.
var hashObject = new
{
request.RequestUri,
RequestorClaims = requestorClaims,
};
string hash = JsonConvert.SerializeObject(hashObject).ComputeHash();
string storageAccountConnectionHash = string.IsNullOrEmpty(_exportJobConfiguration.StorageAccountConnection) ?
string.Empty :
Microsoft.Health.Core.Extensions.StringExtensions.ComputeHash(_exportJobConfiguration.StorageAccountConnection);
// Check to see if a matching job exists or not. If a matching job exists, we will return that instead.
// Otherwise, we will create a new export job. This will be a best effort since the likelihood of this happen should be small.
ExportJobOutcome outcome = await _fhirOperationDataStore.GetExportJobByHashAsync(hash, cancellationToken);
ExportJobFormatConfiguration formatConfiguration = null;
if (request.FormatName != null)
{
formatConfiguration = _exportJobConfiguration.Formats?.FirstOrDefault(
(ExportJobFormatConfiguration formatConfig) => formatConfig.Name.Equals(request.FormatName, StringComparison.OrdinalIgnoreCase));
if (formatConfiguration == null)
{
throw new BadRequestException(Resources.ExportFormatNotFound);
}
}
formatConfiguration ??= _exportJobConfiguration.Formats?.FirstOrDefault(
(ExportJobFormatConfiguration formatConfig) => formatConfig.Default);
formatConfiguration ??= new ExportJobFormatConfiguration()
{
Format = request.ContainerName == null ? ExportFormatTags.ResourceName : $"{ExportFormatTags.Timestamp}-{ExportFormatTags.Id}/{ExportFormatTags.ResourceName}",
};
if (outcome == null)
{
var jobRecord = new ExportJobRecord(
request.RequestUri,
request.RequestType,
formatConfiguration.Format,
request.ResourceType,
hash,
requestorClaims,
request.Since,
request.GroupId,
storageAccountConnectionHash,
_exportJobConfiguration.StorageAccountUri,
request.AnonymizationConfigurationLocation,
request.AnonymizationConfigurationFileETag,
_exportJobConfiguration.MaximumNumberOfResourcesPerQuery,
_exportJobConfiguration.NumberOfPagesPerCommit,
request.ContainerName);
outcome = await _fhirOperationDataStore.CreateExportJobAsync(jobRecord, cancellationToken);
}
return(new CreateExportResponse(outcome.JobRecord.Id));
}
/// <inheritdoc />
public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETag, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(exportJobRecord, nameof(exportJobRecord));
_exportJobRecord = exportJobRecord;
_weakETag = weakETag;
try
{
// Get destination type from secret store and connect to the destination using appropriate client.
await GetDestinationInfoAndConnectAsync(cancellationToken);
// If we are resuming a job, we can detect that by checking the progress info from the job record.
// If it is null, then we know we are processing a new job.
if (_exportJobRecord.Progress == null)
{
_exportJobRecord.Progress = new ExportJobProgress(continuationToken: null, page: 0);
}
ExportJobProgress progress = _exportJobRecord.Progress;
// Current batch will be used to organize a set of search results into a group so that they can be committed together.
uint currentBatchId = progress.Page;
// The first item is placeholder for continuation token so that it can be updated efficiently later.
var queryParameters = new Tuple <string, string>[]
{
Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken),
Tuple.Create(KnownQueryParameterNames.Count, _exportJobConfiguration.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)),
Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{_exportJobRecord.QueuedTime.ToString("o", CultureInfo.InvariantCulture)}"),
};
// Process the export if:
// 1. There is continuation token, which means there is more resource to be exported.
// 2. There is no continuation token but the page is 0, which means it's the initial export.
while (progress.ContinuationToken != null || progress.Page == 0)
{
SearchResult searchResult;
// Search and process the results.
using (IScoped <ISearchService> searchService = _searchServiceFactory())
{
// If the continuation token is null, then we will exclude it. Calculate the offset and count to be passed in.
int offset = queryParameters[0].Item2 == null ? 1 : 0;
searchResult = await searchService.Value.SearchAsync(
_exportJobRecord.ResourceType,
new ArraySegment <Tuple <string, string> >(queryParameters, offset, queryParameters.Length - offset),
cancellationToken);
}
await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, cancellationToken);
if (searchResult.ContinuationToken == null)
{
// No more continuation token, we are done.
break;
}
// Update the continuation token (local cache).
progress.UpdateContinuationToken(searchResult.ContinuationToken);
queryParameters[0] = Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken);
if (progress.Page % _exportJobConfiguration.NumberOfPagesPerCommit == 0)
{
// Commit the changes.
await _exportDestinationClient.CommitAsync(cancellationToken);
// Update the job record.
await UpdateJobRecordAsync(cancellationToken);
currentBatchId = progress.Page;
}
}
// Commit one last time for any pending changes.
await _exportDestinationClient.CommitAsync(cancellationToken);
await CompleteJobAsync(OperationStatus.Completed, cancellationToken);
_logger.LogTrace("Successfully completed the job.");
try
{
// Best effort to delete the secret. If it fails to delete, then move on.
await _secretStore.DeleteSecretAsync(_exportJobRecord.SecretName, cancellationToken);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to delete the secret.");
}
}
catch (JobConflictException)
{
// The export job was updated externally. There might be some additional resources that were exported
// but we will not be updating the job record.
_logger.LogTrace("The job was updated by another process.");
}
catch (SecretStoreException sse)
{
_logger.LogError(sse, "Secret store error. The job will be marked as failed.");
_exportJobRecord.FailureDetails = new ExportJobFailureDetails(sse.Message, sse.ResponseStatusCode);
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
catch (DestinationConnectionException dce)
{
_logger.LogError(dce, "Can't connect to destination. The job will be marked as failed.");
_exportJobRecord.FailureDetails = new ExportJobFailureDetails(dce.Message, dce.StatusCode);
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
catch (Exception ex)
{
// The job has encountered an error it cannot recover from.
// Try to update the job to failed state.
_logger.LogError(ex, "Encountered an unhandled exception. The job will be marked as failed.");
_exportJobRecord.FailureDetails = new ExportJobFailureDetails(Resources.UnknownError, HttpStatusCode.InternalServerError);
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
}
private ExportJobOutcome CreateExportJobOutcome(ExportJobRecord exportJobRecord, WeakETag weakETag = null)
{
return(new ExportJobOutcome(
exportJobRecord,
weakETag ?? WeakETag.FromVersionId("123")));
}
