public async Task GivenDeleteSecretFailed_WhenExecuted_ThenJobStatusShouldBeUpdatedToCompleted() { _searchService.SearchAsync( Arg.Any <string>(), Arg.Any <IReadOnlyList <Tuple <string, string> > >(), _cancellationToken) .Returns(CreateSearchResult()); _secretStore.DeleteSecretAsync(Arg.Any <string>(), _cancellationToken).Returns <SecretWrapper>(_ => throw new Exception()); await _exportJobTask.ExecuteAsync(_exportJobRecord, _weakETag, _cancellationToken); Assert.NotNull(_lastExportJobOutcome); Assert.Equal(OperationStatus.Completed, _lastExportJobOutcome.JobRecord.Status); }
/// <inheritdoc /> public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETag, CancellationToken cancellationToken) { EnsureArg.IsNotNull(exportJobRecord, nameof(exportJobRecord)); _exportJobRecord = exportJobRecord; _weakETag = weakETag; try { // Get destination type from secret store. DestinationInfo destinationInfo = await GetDestinationInfo(cancellationToken); // Connect to the destination using appropriate client. _exportDestinationClient = _exportDestinationClientFactory.Create(destinationInfo.DestinationType); await _exportDestinationClient.ConnectAsync(destinationInfo.DestinationConnectionString, cancellationToken, _exportJobRecord.Id); // TODO: For now, always restart from the beginning. We will support resume in another work item. _exportJobRecord.Progress = new ExportJobProgress(continuationToken: null, page: 0); ExportJobProgress progress = _exportJobRecord.Progress; // Current page will be used to organize a set of search results into a group so that they can be committed together. uint currentBatchId = progress.Page; // The first item is placeholder for continuation token so that it can be updated efficiently later. var queryParameters = new Tuple <string, string>[] { null, Tuple.Create(KnownQueryParameterNames.Count, _exportJobConfiguration.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)), Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{_exportJobRecord.QueuedTime.ToString("o", CultureInfo.InvariantCulture)}"), }; // Process the export if: // 1. There is continuation token, which means there is more resource to be exported. // 2. There is no continuation token but the page is 0, which means it's the initial export. while (progress.ContinuationToken != null || progress.Page == 0) { // Commit the changes if necessary. if (progress.Page != 0 && progress.Page % _exportJobConfiguration.NumberOfPagesPerCommit == 0) { await _exportDestinationClient.CommitAsync(cancellationToken); // Update the job record. await UpdateJobRecord(_exportJobRecord, cancellationToken); currentBatchId = progress.Page; } // Set the continuation token. queryParameters[0] = Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken); SearchResult searchResult = await _searchService.SearchAsync(_exportJobRecord.ResourceType, queryParameters, cancellationToken); foreach (ResourceWrapper resourceWrapper in searchResult.Results) { await ProcessResourceWrapperAsync(resourceWrapper, currentBatchId, cancellationToken); } if (searchResult.ContinuationToken == null) { // No more continuation token, we are done. break; } // Update the job record. progress.UpdateContinuationToken(searchResult.ContinuationToken); } // Commit one last time for any pending changes. await _exportDestinationClient.CommitAsync(cancellationToken); _exportJobRecord.Output.AddRange(_resourceTypeToFileInfoMapping.Values); _logger.LogTrace("Successfully completed the job."); await UpdateJobStatus(OperationStatus.Completed, updateEndTimestamp : true, cancellationToken); try { // Best effort to delete the secret. If it fails to delete, then move on. await _secretStore.DeleteSecretAsync(_exportJobRecord.SecretName, cancellationToken); } catch (Exception ex) { _logger.LogWarning(ex, "Failed to delete the secret."); } } catch (JobConflictException) { // The job was updated by another process. _logger.LogWarning("The job was updated by another process."); // TODO: We will want to get the latest and merge the results without updating the status. return; } catch (Exception ex) { // The job has encountered an error it cannot recover from. // Try to update the job to failed state. _logger.LogError(ex, "Encountered an unhandled exception. The job will be marked as failed."); await UpdateJobStatus(OperationStatus.Failed, updateEndTimestamp : true, cancellationToken); } }
/// <inheritdoc /> public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETag, CancellationToken cancellationToken) { EnsureArg.IsNotNull(exportJobRecord, nameof(exportJobRecord)); _exportJobRecord = exportJobRecord; _weakETag = weakETag; try { // Get destination type from secret store and connect to the destination using appropriate client. await GetDestinationInfoAndConnectAsync(cancellationToken); // If we are resuming a job, we can detect that by checking the progress info from the job record. // If it is null, then we know we are processing a new job. if (_exportJobRecord.Progress == null) { _exportJobRecord.Progress = new ExportJobProgress(continuationToken: null, page: 0); } ExportJobProgress progress = _exportJobRecord.Progress; // Current batch will be used to organize a set of search results into a group so that they can be committed together. uint currentBatchId = progress.Page; // The first item is placeholder for continuation token so that it can be updated efficiently later. var queryParameters = new Tuple <string, string>[] { Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken), Tuple.Create(KnownQueryParameterNames.Count, _exportJobConfiguration.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)), Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{_exportJobRecord.QueuedTime.ToString("o", CultureInfo.InvariantCulture)}"), }; // Process the export if: // 1. There is continuation token, which means there is more resource to be exported. // 2. There is no continuation token but the page is 0, which means it's the initial export. while (progress.ContinuationToken != null || progress.Page == 0) { SearchResult searchResult; // Search and process the results. using (IScoped <ISearchService> searchService = _searchServiceFactory()) { // If the continuation token is null, then we will exclude it. Calculate the offset and count to be passed in. int offset = queryParameters[0].Item2 == null ? 1 : 0; searchResult = await searchService.Value.SearchAsync( _exportJobRecord.ResourceType, new ArraySegment <Tuple <string, string> >(queryParameters, offset, queryParameters.Length - offset), cancellationToken); } await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, cancellationToken); if (searchResult.ContinuationToken == null) { // No more continuation token, we are done. break; } // Update the continuation token (local cache). progress.UpdateContinuationToken(searchResult.ContinuationToken); queryParameters[0] = Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken); if (progress.Page % _exportJobConfiguration.NumberOfPagesPerCommit == 0) { // Commit the changes. await _exportDestinationClient.CommitAsync(cancellationToken); // Update the job record. await UpdateJobRecordAsync(cancellationToken); currentBatchId = progress.Page; } } // Commit one last time for any pending changes. await _exportDestinationClient.CommitAsync(cancellationToken); await CompleteJobAsync(OperationStatus.Completed, cancellationToken); _logger.LogTrace("Successfully completed the job."); try { // Best effort to delete the secret. If it fails to delete, then move on. await _secretStore.DeleteSecretAsync(_exportJobRecord.SecretName, cancellationToken); } catch (Exception ex) { _logger.LogWarning(ex, "Failed to delete the secret."); } } catch (JobConflictException) { // The export job was updated externally. There might be some additional resources that were exported // but we will not be updating the job record. _logger.LogTrace("The job was updated by another process."); } catch (Exception ex) { // The job has encountered an error it cannot recover from. // Try to update the job to failed state. _logger.LogError(ex, "Encountered an unhandled exception. The job will be marked as failed."); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } }