public ExportJobTaskTests() { _cancellationToken = _cancellationTokenSource.Token; _exportJobRecord = new ExportJobRecord( new Uri("https://localhost/ExportJob/"), "Patient", "hash"); _fhirOperationDataStore.UpdateExportJobAsync(_exportJobRecord, _weakETag, _cancellationToken).Returns(x => { _lastExportJobOutcome = new ExportJobOutcome(_exportJobRecord, _weakETag); return(_lastExportJobOutcome); }); _secretStore.GetSecretAsync(Arg.Any <string>(), _cancellationToken).Returns(x => new SecretWrapper(x.ArgAt <string>(0), "{\"destinationType\": \"in-memory\"}")); _exportDestinationClientFactory.Create("in-memory").Returns(_inMemoryDestinationClient); _resourceToByteArraySerializer.Serialize(Arg.Any <ResourceWrapper>()).Returns(x => Encoding.UTF8.GetBytes(x.ArgAt <ResourceWrapper>(0).ResourceId)); _exportJobTask = new ExportJobTask( () => _fhirOperationDataStore.CreateMockScope(), _secretStore, Options.Create(_exportJobConfiguration), () => _searchService.CreateMockScope(), _resourceToByteArraySerializer, _exportDestinationClientFactory, NullLogger <ExportJobTask> .Instance); }
// Get destination info from secret store, create appropriate export client and connect to destination. private async Task GetDestinationInfoAndConnectAsync(CancellationToken cancellationToken) { SecretWrapper secret = await _secretStore.GetSecretAsync(_exportJobRecord.SecretName, cancellationToken); DestinationInfo destinationInfo = JsonConvert.DeserializeObject <DestinationInfo>(secret.SecretValue); _exportDestinationClient = _exportDestinationClientFactory.Create(destinationInfo.DestinationType); await _exportDestinationClient.ConnectAsync(destinationInfo.DestinationConnectionString, cancellationToken, _exportJobRecord.Id); }
/// <inheritdoc /> public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETag, CancellationToken cancellationToken) { EnsureArg.IsNotNull(exportJobRecord, nameof(exportJobRecord)); _exportJobRecord = exportJobRecord; _weakETag = weakETag; try { // Get destination type from secret store. DestinationInfo destinationInfo = await GetDestinationInfo(cancellationToken); // Connect to the destination using appropriate client. _exportDestinationClient = _exportDestinationClientFactory.Create(destinationInfo.DestinationType); await _exportDestinationClient.ConnectAsync(destinationInfo.DestinationConnectionString, cancellationToken, _exportJobRecord.Id); // TODO: For now, always restart from the beginning. We will support resume in another work item. _exportJobRecord.Progress = new ExportJobProgress(continuationToken: null, page: 0); ExportJobProgress progress = _exportJobRecord.Progress; // Current page will be used to organize a set of search results into a group so that they can be committed together. uint currentBatchId = progress.Page; // The first item is placeholder for continuation token so that it can be updated efficiently later. var queryParameters = new Tuple <string, string>[] { null, Tuple.Create(KnownQueryParameterNames.Count, _exportJobConfiguration.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)), Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{_exportJobRecord.QueuedTime.ToString("o", CultureInfo.InvariantCulture)}"), }; // Process the export if: // 1. There is continuation token, which means there is more resource to be exported. // 2. There is no continuation token but the page is 0, which means it's the initial export. while (progress.ContinuationToken != null || progress.Page == 0) { // Commit the changes if necessary. if (progress.Page != 0 && progress.Page % _exportJobConfiguration.NumberOfPagesPerCommit == 0) { await _exportDestinationClient.CommitAsync(cancellationToken); // Update the job record. await UpdateJobRecord(_exportJobRecord, cancellationToken); currentBatchId = progress.Page; } // Set the continuation token. queryParameters[0] = Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken); SearchResult searchResult = await _searchService.SearchAsync(_exportJobRecord.ResourceType, queryParameters, cancellationToken); foreach (ResourceWrapper resourceWrapper in searchResult.Results) { await ProcessResourceWrapperAsync(resourceWrapper, currentBatchId, cancellationToken); } if (searchResult.ContinuationToken == null) { // No more continuation token, we are done. break; } // Update the job record. progress.UpdateContinuationToken(searchResult.ContinuationToken); } // Commit one last time for any pending changes. await _exportDestinationClient.CommitAsync(cancellationToken); _exportJobRecord.Output.AddRange(_resourceTypeToFileInfoMapping.Values); _logger.LogTrace("Successfully completed the job."); await UpdateJobStatus(OperationStatus.Completed, updateEndTimestamp : true, cancellationToken); try { // Best effort to delete the secret. If it fails to delete, then move on. await _secretStore.DeleteSecretAsync(_exportJobRecord.SecretName, cancellationToken); } catch (Exception ex) { _logger.LogWarning(ex, "Failed to delete the secret."); } } catch (JobConflictException) { // The job was updated by another process. _logger.LogWarning("The job was updated by another process."); // TODO: We will want to get the latest and merge the results without updating the status. return; } catch (Exception ex) { // The job has encountered an error it cannot recover from. // Try to update the job to failed state. _logger.LogError(ex, "Encountered an unhandled exception. The job will be marked as failed."); await UpdateJobStatus(OperationStatus.Failed, updateEndTimestamp : true, cancellationToken); } }