private async Task <SearchResult> ExecuteReindexQueryAsync(ReindexJobQueryStatus queryStatus, bool countOnly, CancellationToken cancellationToken)
{
var queryParametersList = new List <Tuple <string, string> >()
{
Tuple.Create(KnownQueryParameterNames.Count, _reindexJobConfiguration.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)),
Tuple.Create(KnownQueryParameterNames.Type, queryStatus.ResourceType),
};
if (queryStatus.ContinuationToken != null)
{
queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, queryStatus.ContinuationToken));
}
if (!_reindexJobRecord.ResourceTypeSearchParameterHashMap.TryGetValue(queryStatus.ResourceType, out string searchParameterHash))
{
searchParameterHash = string.Empty;
}
using (IScoped <ISearchService> searchService = _searchServiceFactory())
{
try
{
return(await searchService.Value.SearchForReindexAsync(queryParametersList, searchParameterHash, countOnly, cancellationToken));
}
catch (Exception ex)
{
_logger.LogError(ex, "Error running reindex query.");
queryStatus.Error = ex.Message;
throw;
}
}
}
private async Task CalculateTotalAndResourceCounts(CancellationToken cancellationToken)
{
int totalCount = 0;
foreach (string resourceType in _reindexJobRecord.Resources)
{
var queryForCount = new ReindexJobQueryStatus(resourceType, continuationToken: null)
{
LastModified = Clock.UtcNow,
Status = OperationStatus.Queued,
};
// update the complete total
SearchResult countOnlyResults = await ExecuteReindexQueryAsync(queryForCount, countOnly : true, cancellationToken);
if (countOnlyResults?.TotalCount != null)
{
// No action needs to be taken if an entry for this resource fails to get added to the dictionary
// We will reindex all resource types that do not have a dictionary entry
_reindexJobRecord.ResourceCounts.TryAdd(resourceType, countOnlyResults.TotalCount.Value);
totalCount += countOnlyResults.TotalCount.Value;
}
else
{
_reindexJobRecord.ResourceCounts.TryAdd(resourceType, 0);
}
}
_reindexJobRecord.Count = totalCount;
}
private async Task <SearchResult> ExecuteReindexQueryAsync(ReindexJobQueryStatus queryStatus, bool countOnly, CancellationToken cancellationToken)
{
var queryParametersList = new List <Tuple <string, string> >()
{
Tuple.Create(KnownQueryParameterNames.Count, _throttleController.GetThrottleBatchSize().ToString(CultureInfo.InvariantCulture)),
Tuple.Create(KnownQueryParameterNames.Type, queryStatus.ResourceType),
};
if (queryStatus.ContinuationToken != null)
{
queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, queryStatus.ContinuationToken));
}
if (!_reindexJobRecord.ResourceTypeSearchParameterHashMap.TryGetValue(queryStatus.ResourceType, out string searchParameterHash))
{
searchParameterHash = string.Empty;
}
using (IScoped <ISearchService> searchService = _searchServiceFactory())
{
try
{
return(await searchService.Value.SearchForReindexAsync(queryParametersList, searchParameterHash, countOnly, cancellationToken));
}
catch (Exception ex)
{
var message = $"Error running reindex query for resource type {queryStatus.ResourceType}.";
var reindexJobException = new ReindexJobException(message, ex);
_logger.LogError(ex, message);
queryStatus.Error = reindexJobException.Message + " : " + ex.Message;
throw reindexJobException;
}
}
}
private async Task ProcessQueryAsync(ReindexJobQueryStatus query, CancellationToken cancellationToken)
{
try
{
query.Status = OperationStatus.Running;
query.LastModified = DateTimeOffset.UtcNow;
// Query first batch of resources
var results = await ExecuteReindexQueryAsync(query, false, cancellationToken);
// if continuation token then update next query
if (!string.IsNullOrEmpty(results.ContinuationToken))
{
var nextQuery = new ReindexJobQueryStatus(results.ContinuationToken);
nextQuery.LastModified = DateTimeOffset.UtcNow;
nextQuery.Status = OperationStatus.Queued;
_reindexJobRecord.QueryList.Add(nextQuery);
}
await UpdateJobAsync(cancellationToken);
// TODO: Release lock on job document so another thread may pick up the next query.
await _updateIndices.ProcessSearchResultsAsync(results, _reindexJobRecord.Hash, cancellationToken);
// TODO: reaquire document lock and update _etag
query.Status = OperationStatus.Completed;
await UpdateJobAsync(cancellationToken);
await CheckJobCompletionStatus(cancellationToken);
}
catch (Exception ex)
{
query.Error = ex.Message;
query.FailureCount++;
_logger.LogError(ex, $"Encountered an unhandled exception. The query failure count increased to {_reindexJobRecord.FailureCount}.");
if (query.FailureCount >= _reindexJobConfiguration.ConsecutiveFailuresThreshold)
{
query.Status = OperationStatus.Failed;
}
else
{
query.Status = OperationStatus.Queued;
}
await UpdateJobAsync(cancellationToken);
}
}
private async Task CalculateTotalCount(CancellationToken cancellationToken)
{
int totalCount = 0;
foreach (string resourceType in _reindexJobRecord.Resources)
{
var queryForCount = new ReindexJobQueryStatus(resourceType, continuationToken: null)
{
LastModified = Clock.UtcNow,
Status = OperationStatus.Queued,
};
// update the complete total
SearchResult countOnlyResults = await ExecuteReindexQueryAsync(queryForCount, countOnly : true, cancellationToken);
totalCount += countOnlyResults.TotalCount.Value;
}
_reindexJobRecord.Count = totalCount;
}
private async Task <ReindexJobQueryStatus> HandleQueryException(ReindexJobQueryStatus query, Exception ex, bool isFhirException, CancellationToken cancellationToken)
{
await _jobSemaphore.WaitAsync(cancellationToken);
try
{
query.Error = ex.Message;
query.FailureCount++;
_logger.LogError(ex, "Encountered an unhandled exception. The query failure count increased to {failureCount}.", _reindexJobRecord.FailureCount);
if (query.FailureCount >= _reindexJobConfiguration.ConsecutiveFailuresThreshold)
{
if (isFhirException)
{
var issue = new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Error,
OperationOutcomeConstants.IssueType.Exception,
ex.Message);
_reindexJobRecord.Error.Add(issue);
}
query.Status = OperationStatus.Failed;
}
else
{
query.Status = OperationStatus.Queued;
}
await UpdateJobAsync();
}
finally
{
_jobSemaphore.Release();
}
return(query);
}
private async Task <(int totalCount, List <string> resourcesTypes)> CalculateTotalCount()
{
int totalCount = 0;
var resourcesTypes = new List <string>();
foreach (string resourceType in _reindexJobRecord.Resources)
{
var queryForCount = new ReindexJobQueryStatus(resourceType, continuationToken: null)
{
LastModified = Clock.UtcNow,
Status = OperationStatus.Queued,
};
SearchResult countOnlyResults = await ExecuteReindexQueryAsync(queryForCount, countOnly : true, _cancellationToken);
if (countOnlyResults?.TotalCount != null && countOnlyResults.TotalCount.Value > 0)
{
totalCount += countOnlyResults.TotalCount.Value;
resourcesTypes.Add(resourceType);
}
}
return(totalCount, resourcesTypes);
}
/// <inheritdoc />
public async Task ExecuteAsync(ReindexJobRecord reindexJobRecord, WeakETag weakETag, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(reindexJobRecord, nameof(reindexJobRecord));
EnsureArg.IsNotNull(weakETag, nameof(weakETag));
_reindexJobRecord = reindexJobRecord;
_weakETag = weakETag;
var jobSemaphore = new SemaphoreSlim(1, 1);
try
{
if (_reindexJobRecord.Status != OperationStatus.Running ||
_reindexJobRecord.StartTime == null)
{
// update job record to running
_reindexJobRecord.Status = OperationStatus.Running;
_reindexJobRecord.StartTime = Clock.UtcNow;
await UpdateJobAsync(cancellationToken);
}
// If we are resuming a job, we can detect that by checking the progress info from the job record.
// If no queries have been added to the progress then this is a new job
if (_reindexJobRecord.QueryList?.Count == 0)
{
// Build query based on new search params
// Find supported, but not yet searchable params
var notYetIndexedParams = _supportedSearchParameterDefinitionManager.GetSupportedButNotSearchableParams();
// if there are not any parameters which are supported but not yet indexed, then we have nothing to do
if (!notYetIndexedParams.Any())
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Information,
OperationOutcomeConstants.IssueType.Informational,
Resources.NoSearchParametersNeededToBeIndexed));
_reindexJobRecord.CanceledTime = DateTimeOffset.UtcNow;
await CompleteJobAsync(OperationStatus.Canceled, cancellationToken);
return;
}
// From the param list, get the list of necessary resources which should be
// included in our query
var resourceList = new HashSet <string>();
foreach (var param in notYetIndexedParams)
{
if (param.TargetResourceTypes != null)
{
resourceList.UnionWith(param.TargetResourceTypes);
}
if (param.BaseResourceTypes != null)
{
resourceList.UnionWith(param.BaseResourceTypes);
}
}
_reindexJobRecord.Resources.AddRange(resourceList);
_reindexJobRecord.SearchParams.AddRange(notYetIndexedParams.Select(p => p.Url.ToString()));
await CalculateTotalCount(cancellationToken);
if (_reindexJobRecord.Count == 0)
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Information,
OperationOutcomeConstants.IssueType.Informational,
Resources.NoResourcesNeedToBeReindexed));
await UpdateParametersAndCompleteJob(cancellationToken);
return;
}
// Generate separate queries for each resource type and add them to query list.
foreach (string resourceType in _reindexJobRecord.Resources)
{
var query = new ReindexJobQueryStatus(resourceType, continuationToken: null)
{
LastModified = Clock.UtcNow,
Status = OperationStatus.Queued,
};
_reindexJobRecord.QueryList.Add(query);
}
await UpdateJobAsync(cancellationToken);
}
var queryTasks = new List <Task <ReindexJobQueryStatus> >();
var queryCancellationTokens = new Dictionary <ReindexJobQueryStatus, CancellationTokenSource>();
// while not all queries are finished
while (_reindexJobRecord.QueryList.Where(q =>
q.Status == OperationStatus.Queued ||
q.Status == OperationStatus.Running).Any())
{
if (_reindexJobRecord.QueryList.Where(q => q.Status == OperationStatus.Queued).Any())
{
// grab the next query from the list which is labeled as queued and run it
var query = _reindexJobRecord.QueryList.Where(q => q.Status == OperationStatus.Queued).OrderBy(q => q.LastModified).FirstOrDefault();
CancellationTokenSource queryTokensSource = new CancellationTokenSource();
queryCancellationTokens.Add(query, queryTokensSource);
#pragma warning disable CS4014 // Suppressed as we want to continue execution and begin processing the next query while this continues to run
queryTasks.Add(ProcessQueryAsync(query, jobSemaphore, queryTokensSource.Token));
#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
_logger.LogInformation($"Reindex job task created {queryTasks.Count} Tasks");
}
// reset stale queries to pending
var staleQueries = _reindexJobRecord.QueryList.Where(
q => q.Status == OperationStatus.Running && q.LastModified < Clock.UtcNow - _reindexJobConfiguration.JobHeartbeatTimeoutThreshold);
foreach (var staleQuery in staleQueries)
{
await jobSemaphore.WaitAsync();
try
{
// if this query has a created task, cancel it
if (queryCancellationTokens.TryGetValue(staleQuery, out var tokenSource))
{
try
{
tokenSource.Cancel(false);
}
catch
{
// may throw exception if the task is disposed
}
}
staleQuery.Status = OperationStatus.Queued;
await UpdateJobAsync(cancellationToken);
}
finally
{
jobSemaphore.Release();
}
}
await Task.Delay(_reindexJobConfiguration.QueryDelayIntervalInMilliseconds);
// Remove all finished tasks from the collections of tasks
// and cancellationTokens
if (queryTasks.Count >= reindexJobRecord.MaximumConcurrency)
{
var taskArray = queryTasks.ToArray();
Task.WaitAny(taskArray);
var finishedTasks = queryTasks.Where(t => t.IsCompleted).ToArray();
foreach (var finishedTask in finishedTasks)
{
queryTasks.Remove(finishedTask);
queryCancellationTokens.Remove(await finishedTask);
}
}
// if our received CancellationToken is cancelled we should
// pass that cancellation request onto all the cancellationTokens
// for the currently executing threads
if (cancellationToken.IsCancellationRequested)
{
foreach (var tokenSource in queryCancellationTokens.Values)
{
tokenSource.Cancel(false);
}
}
}
Task.WaitAll(queryTasks.ToArray());
await jobSemaphore.WaitAsync();
try
{
await CheckJobCompletionStatus(cancellationToken);
}
finally
{
jobSemaphore.Release();
}
}
catch (JobConflictException)
{
// The reindex job was updated externally.
_logger.LogInformation("The job was updated by another process.");
}
catch (Exception ex)
{
await jobSemaphore.WaitAsync();
try
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Error,
OperationOutcomeConstants.IssueType.Exception,
ex.Message));
_reindexJobRecord.FailureCount++;
_logger.LogError(ex, $"Encountered an unhandled exception. The job failure count increased to {_reindexJobRecord.FailureCount}.");
await UpdateJobAsync(cancellationToken);
if (_reindexJobRecord.FailureCount >= _reindexJobConfiguration.ConsecutiveFailuresThreshold)
{
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
else
{
_reindexJobRecord.Status = OperationStatus.Queued;
await UpdateJobAsync(cancellationToken);
}
}
finally
{
jobSemaphore.Release();
}
}
finally
{
jobSemaphore.Dispose();
}
}
private async Task <ReindexJobQueryStatus> ProcessQueryAsync(ReindexJobQueryStatus query, SemaphoreSlim jobSemaphore, CancellationToken cancellationToken)
{
try
{
SearchResult results;
await jobSemaphore.WaitAsync();
try
{
query.Status = OperationStatus.Running;
query.LastModified = DateTimeOffset.UtcNow;
// Query first batch of resources
results = await ExecuteReindexQueryAsync(query, countOnly : false, cancellationToken);
// if continuation token then update next query
if (!string.IsNullOrEmpty(results?.ContinuationToken))
{
var encodedContinuationToken = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(results.ContinuationToken));
var nextQuery = new ReindexJobQueryStatus(query.ResourceType, encodedContinuationToken)
{
LastModified = Clock.UtcNow,
Status = OperationStatus.Queued,
};
_reindexJobRecord.QueryList.Add(nextQuery);
}
await UpdateJobAsync(cancellationToken);
}
finally
{
jobSemaphore.Release();
}
_logger.LogInformation($"Reindex job current thread: {Thread.CurrentThread.ManagedThreadId}");
await _reindexUtilities.ProcessSearchResultsAsync(results, _reindexJobRecord.ResourceTypeSearchParameterHashMap, cancellationToken);
if (!cancellationToken.IsCancellationRequested)
{
await jobSemaphore.WaitAsync();
try
{
_reindexJobRecord.Progress += results.Results.Count();
query.Status = OperationStatus.Completed;
await UpdateJobAsync(cancellationToken);
}
finally
{
jobSemaphore.Release();
}
}
return(query);
}
catch (Exception ex)
{
await jobSemaphore.WaitAsync();
try
{
query.Error = ex.Message;
query.FailureCount++;
_logger.LogError(ex, $"Encountered an unhandled exception. The query failure count increased to {_reindexJobRecord.FailureCount}.");
if (query.FailureCount >= _reindexJobConfiguration.ConsecutiveFailuresThreshold)
{
query.Status = OperationStatus.Failed;
}
else
{
query.Status = OperationStatus.Queued;
}
await UpdateJobAsync(cancellationToken);
}
finally
{
jobSemaphore.Release();
}
return(query);
}
}
/// <inheritdoc />
public async Task ExecuteAsync(ReindexJobRecord reindexJobRecord, WeakETag weakETag, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(reindexJobRecord, nameof(reindexJobRecord));
EnsureArg.IsNotNull(weakETag, nameof(weakETag));
_reindexJobRecord = reindexJobRecord;
_weakETag = weakETag;
var jobSemaphore = new SemaphoreSlim(1, 1);
var existingFhirRequestContext = _contextAccessor.FhirRequestContext;
try
{
// Add a request context so Datastore consumption can be added
var fhirRequestContext = new FhirRequestContext(
method: OperationsConstants.Reindex,
uriString: "$reindex",
baseUriString: "$reindex",
correlationId: _reindexJobRecord.Id,
requestHeaders: new Dictionary <string, StringValues>(),
responseHeaders: new Dictionary <string, StringValues>())
{
IsBackgroundTask = true,
AuditEventType = OperationsConstants.Reindex,
};
_contextAccessor.FhirRequestContext = fhirRequestContext;
using (IScoped <IFhirDataStore> store = _fhirDataStoreFactory())
{
var provisionedCapacity = await store.Value.GetProvisionedDataStoreCapacityAsync(cancellationToken);
_throttleController.Initialize(_reindexJobRecord, provisionedCapacity);
}
if (_reindexJobRecord.Status != OperationStatus.Running ||
_reindexJobRecord.StartTime == null)
{
// update job record to running
_reindexJobRecord.Status = OperationStatus.Running;
_reindexJobRecord.StartTime = Clock.UtcNow;
await UpdateJobAsync(cancellationToken);
}
// If we are resuming a job, we can detect that by checking the progress info from the job record.
// If no queries have been added to the progress then this is a new job
if (_reindexJobRecord.QueryList?.Count == 0)
{
// Build query based on new search params
// Find supported, but not yet searchable params
var notYetIndexedParams = _supportedSearchParameterDefinitionManager.GetSearchParametersRequiringReindexing();
// if there are not any parameters which are supported but not yet indexed, then we have nothing to do
if (!notYetIndexedParams.Any())
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Information,
OperationOutcomeConstants.IssueType.Informational,
Resources.NoSearchParametersNeededToBeIndexed));
_reindexJobRecord.CanceledTime = DateTimeOffset.UtcNow;
await CompleteJobAsync(OperationStatus.Canceled, cancellationToken);
return;
}
// From the param list, get the list of necessary resources which should be
// included in our query
var resourceList = new HashSet <string>();
foreach (var param in notYetIndexedParams)
{
foreach (var baseResourceType in param.BaseResourceTypes)
{
if (baseResourceType == KnownResourceTypes.Resource)
{
resourceList.UnionWith(_modelInfoProvider.GetResourceTypeNames().ToHashSet());
// We added all possible resource types, so no need to continue
break;
}
if (baseResourceType == KnownResourceTypes.DomainResource)
{
var domainResourceChildResourceTypes = _modelInfoProvider.GetResourceTypeNames().ToHashSet();
// Remove types that inherit from Resource directly
domainResourceChildResourceTypes.Remove(KnownResourceTypes.Binary);
domainResourceChildResourceTypes.Remove(KnownResourceTypes.Bundle);
domainResourceChildResourceTypes.Remove(KnownResourceTypes.Parameters);
resourceList.UnionWith(domainResourceChildResourceTypes);
}
else
{
resourceList.UnionWith(new[] { baseResourceType });
}
}
}
_reindexJobRecord.Resources.AddRange(resourceList);
_reindexJobRecord.SearchParams.AddRange(notYetIndexedParams.Select(p => p.Url.ToString()));
await CalculateTotalAndResourceCounts(cancellationToken);
if (_reindexJobRecord.Count == 0)
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Information,
OperationOutcomeConstants.IssueType.Informational,
Resources.NoResourcesNeedToBeReindexed));
await UpdateParametersAndCompleteJob(cancellationToken);
return;
}
// Generate separate queries for each resource type and add them to query list.
foreach (string resourceType in _reindexJobRecord.Resources)
{
// Checking resource specific counts is a performance improvement,
// so if an entry for this resource failed to get added to the count dictionary, run a query anyways
if (!_reindexJobRecord.ResourceCounts.ContainsKey(resourceType) || _reindexJobRecord.ResourceCounts[resourceType] > 0)
{
var query = new ReindexJobQueryStatus(resourceType, continuationToken: null)
{
LastModified = Clock.UtcNow,
Status = OperationStatus.Queued,
};
_reindexJobRecord.QueryList.TryAdd(query, 1);
}
}
await UpdateJobAsync(cancellationToken);
_throttleController.UpdateDatastoreUsage();
}
var queryTasks = new List <Task <ReindexJobQueryStatus> >();
var queryCancellationTokens = new Dictionary <ReindexJobQueryStatus, CancellationTokenSource>();
// while not all queries are finished
while (_reindexJobRecord.QueryList.Keys.Where(q =>
q.Status == OperationStatus.Queued ||
q.Status == OperationStatus.Running).Any())
{
if (_reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Queued).Any())
{
// grab the next query from the list which is labeled as queued and run it
var query = _reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Queued).OrderBy(q => q.LastModified).FirstOrDefault();
CancellationTokenSource queryTokensSource = new CancellationTokenSource();
queryCancellationTokens.TryAdd(query, queryTokensSource);
#pragma warning disable CS4014 // Suppressed as we want to continue execution and begin processing the next query while this continues to run
queryTasks.Add(ProcessQueryAsync(query, jobSemaphore, queryTokensSource.Token));
#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
_logger.LogInformation($"Reindex job task created {queryTasks.Count} Tasks");
}
// reset stale queries to pending
var staleQueries = _reindexJobRecord.QueryList.Keys.Where(
q => q.Status == OperationStatus.Running && q.LastModified < Clock.UtcNow - _reindexJobConfiguration.JobHeartbeatTimeoutThreshold);
foreach (var staleQuery in staleQueries)
{
await jobSemaphore.WaitAsync();
try
{
// if this query has a created task, cancel it
if (queryCancellationTokens.TryGetValue(staleQuery, out var tokenSource))
{
try
{
tokenSource.Cancel(false);
}
catch
{
// may throw exception if the task is disposed
}
}
staleQuery.Status = OperationStatus.Queued;
await UpdateJobAsync(cancellationToken);
}
finally
{
jobSemaphore.Release();
}
}
var averageDbConsumption = _throttleController.UpdateDatastoreUsage();
_logger.LogInformation($"Reindex avaerage DB consumption: {averageDbConsumption}");
var throttleDelayTime = _throttleController.GetThrottleBasedDelay();
_logger.LogInformation($"Reindex throttle delay: {throttleDelayTime}");
await Task.Delay(_reindexJobRecord.QueryDelayIntervalInMilliseconds + throttleDelayTime);
// Remove all finished tasks from the collections of tasks
// and cancellationTokens
if (queryTasks.Count >= reindexJobRecord.MaximumConcurrency)
{
var taskArray = queryTasks.ToArray();
Task.WaitAny(taskArray);
var finishedTasks = queryTasks.Where(t => t.IsCompleted).ToArray();
foreach (var finishedTask in finishedTasks)
{
queryTasks.Remove(finishedTask);
queryCancellationTokens.Remove(await finishedTask);
}
}
// if our received CancellationToken is cancelled we should
// pass that cancellation request onto all the cancellationTokens
// for the currently executing threads
if (cancellationToken.IsCancellationRequested)
{
foreach (var tokenSource in queryCancellationTokens.Values)
{
tokenSource.Cancel(false);
}
}
}
Task.WaitAll(queryTasks.ToArray());
await jobSemaphore.WaitAsync();
try
{
await CheckJobCompletionStatus(cancellationToken);
}
finally
{
jobSemaphore.Release();
}
}
catch (JobConflictException)
{
// The reindex job was updated externally.
_logger.LogInformation("The job was updated by another process.");
}
catch (Exception ex)
{
await jobSemaphore.WaitAsync();
try
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Error,
OperationOutcomeConstants.IssueType.Exception,
ex.Message));
_reindexJobRecord.FailureCount++;
_logger.LogError(ex, "Encountered an unhandled exception. The job failure count increased to {failureCount}.", _reindexJobRecord.FailureCount);
await UpdateJobAsync(cancellationToken);
if (_reindexJobRecord.FailureCount >= _reindexJobConfiguration.ConsecutiveFailuresThreshold)
{
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
else
{
_reindexJobRecord.Status = OperationStatus.Queued;
await UpdateJobAsync(cancellationToken);
}
}
finally
{
jobSemaphore.Release();
}
}
finally
{
jobSemaphore.Dispose();
_contextAccessor.FhirRequestContext = existingFhirRequestContext;
}
}
private async Task <ReindexJobQueryStatus> ProcessQueryAsync(ReindexJobQueryStatus query, SemaphoreSlim jobSemaphore, CancellationToken cancellationToken)
{
try
{
SearchResult results;
await jobSemaphore.WaitAsync();
try
{
query.Status = OperationStatus.Running;
query.LastModified = DateTimeOffset.UtcNow;
// Query first batch of resources
results = await ExecuteReindexQueryAsync(query, countOnly : false, cancellationToken);
// if continuation token then update next query
if (!string.IsNullOrEmpty(results?.ContinuationToken))
{
var encodedContinuationToken = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(results.ContinuationToken));
var nextQuery = new ReindexJobQueryStatus(query.ResourceType, encodedContinuationToken)
{
LastModified = Clock.UtcNow,
Status = OperationStatus.Queued,
};
_reindexJobRecord.QueryList.TryAdd(nextQuery, 1);
}
await UpdateJobAsync(cancellationToken);
_throttleController.UpdateDatastoreUsage();
}
finally
{
jobSemaphore.Release();
}
_logger.LogInformation($"Reindex job current thread: {Thread.CurrentThread.ManagedThreadId}");
await _reindexUtilities.ProcessSearchResultsAsync(results, _reindexJobRecord.ResourceTypeSearchParameterHashMap, cancellationToken);
_throttleController.UpdateDatastoreUsage();
if (!cancellationToken.IsCancellationRequested)
{
await jobSemaphore.WaitAsync();
try
{
_logger.LogInformation("Reindex job updating progress, current result count: {0}", results.Results.Count());
_reindexJobRecord.Progress += results.Results.Count();
query.Status = OperationStatus.Completed;
// Remove oldest completed queryStatus object if count > 10
// to ensure reindex job document doesn't grow too large
if (_reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Completed).Count() > 10)
{
var queryStatusToRemove = _reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Completed).OrderBy(q => q.LastModified).FirstOrDefault();
_reindexJobRecord.QueryList.TryRemove(queryStatusToRemove, out var removedByte);
}
await UpdateJobAsync(cancellationToken);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Reindex error occurred recording progress.");
throw;
}
finally
{
jobSemaphore.Release();
}
}
return(query);
}
catch (Exception ex)
{
await jobSemaphore.WaitAsync();
try
{
query.Error = ex.Message;
query.FailureCount++;
_logger.LogError(ex, "Encountered an unhandled exception. The query failure count increased to {failureCount}.", _reindexJobRecord.FailureCount);
if (query.FailureCount >= _reindexJobConfiguration.ConsecutiveFailuresThreshold)
{
query.Status = OperationStatus.Failed;
}
else
{
query.Status = OperationStatus.Queued;
}
await UpdateJobAsync(cancellationToken);
}
finally
{
jobSemaphore.Release();
}
return(query);
}
}
private async Task <ReindexJobQueryStatus> ProcessQueryAsync(ReindexJobQueryStatus query, CancellationToken cancellationToken)
{
try
{
SearchResult results;
await _jobSemaphore.WaitAsync(cancellationToken);
try
{
// Query first batch of resources
results = await ExecuteReindexQueryAsync(query, countOnly : false, cancellationToken);
// If continuation token then update next query but only if parent query haven't been in pipeline.
// For cases like retry or stale query we don't want to start another chain.
if (!string.IsNullOrEmpty(results?.ContinuationToken) && !query.CreatedChild)
{
var encodedContinuationToken = ContinuationTokenConverter.Encode(results.ContinuationToken);
var nextQuery = new ReindexJobQueryStatus(query.ResourceType, encodedContinuationToken)
{
LastModified = Clock.UtcNow,
Status = OperationStatus.Queued,
};
_reindexJobRecord.QueryList.TryAdd(nextQuery, 1);
query.CreatedChild = true;
}
await UpdateJobAsync();
_throttleController.UpdateDatastoreUsage();
}
finally
{
_jobSemaphore.Release();
}
_logger.LogInformation($"Reindex job current thread: {Thread.CurrentThread.ManagedThreadId}");
await _reindexUtilities.ProcessSearchResultsAsync(results, _reindexJobRecord.ResourceTypeSearchParameterHashMap, cancellationToken);
_throttleController.UpdateDatastoreUsage();
if (!_cancellationToken.IsCancellationRequested)
{
await _jobSemaphore.WaitAsync(cancellationToken);
try
{
_logger.LogInformation("Reindex job updating progress, current result count: {0}", results.Results.Count());
_reindexJobRecord.Progress += results.Results.Count();
query.Status = OperationStatus.Completed;
// Remove oldest completed queryStatus object if count > 10
// to ensure reindex job document doesn't grow too large
if (_reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Completed).Count() > 10)
{
var queryStatusToRemove = _reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Completed).OrderBy(q => q.LastModified).FirstOrDefault();
_reindexJobRecord.QueryList.TryRemove(queryStatusToRemove, out var removedByte);
}
await UpdateJobAsync();
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Reindex error occurred recording progress.");
throw;
}
finally
{
_jobSemaphore.Release();
}
}
return(query);
}
catch (FhirException ex)
{
return(await HandleQueryException(query, ex, true, cancellationToken));
}
catch (Exception ex)
{
return(await HandleQueryException(query, ex, false, cancellationToken));
}
}
private async Task <bool> TryPopulateNewJobFields()
{
// Build query based on new search params
// Find supported, but not yet searchable params
var possibleNotYetIndexedParams = _supportedSearchParameterDefinitionManager.GetSearchParametersRequiringReindexing();
var notYetIndexedParams = new List <SearchParameterInfo>();
var resourceList = new HashSet <string>();
// filter list of SearchParameters by the target resource types
if (_reindexJobRecord.TargetResourceTypes.Any())
{
foreach (var searchParam in possibleNotYetIndexedParams)
{
var searchParamResourceTypes = GetDerivedResourceTypes(searchParam.BaseResourceTypes);
var matchingResourceTypes = searchParamResourceTypes.Intersect(_reindexJobRecord.TargetResourceTypes);
if (matchingResourceTypes.Any())
{
notYetIndexedParams.Add(searchParam);
// add matching resource types to the set of resource types which we will reindex
resourceList.UnionWith(matchingResourceTypes);
}
else
{
_logger.LogInformation("Search parameter {url} is not being reindexed as it does not match the target types of reindex job {reindexid}.", searchParam.Url, _reindexJobRecord.Id);
}
}
}
else
{
notYetIndexedParams.AddRange(possibleNotYetIndexedParams);
// From the param list, get the list of necessary resources which should be
// included in our query
foreach (var param in notYetIndexedParams)
{
var searchParamResourceTypes = GetDerivedResourceTypes(param.BaseResourceTypes);
resourceList.UnionWith(searchParamResourceTypes);
}
}
// if there are not any parameters which are supported but not yet indexed, then we have nothing to do
if (!notYetIndexedParams.Any())
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Information,
OperationOutcomeConstants.IssueType.Informational,
Core.Resources.NoSearchParametersNeededToBeIndexed));
_reindexJobRecord.CanceledTime = Clock.UtcNow;
await MoveToFinalStatusAsync(OperationStatus.Canceled);
return(false);
}
// Save the list of resource types in the reindexjob document
foreach (var resource in resourceList)
{
_reindexJobRecord.Resources.Add(resource);
}
// save the list of search parameters to the reindexjob document
foreach (var searchParams in notYetIndexedParams.Select(p => p.Url.OriginalString))
{
_reindexJobRecord.SearchParams.Add(searchParams);
}
await CalculateAndSetTotalAndResourceCounts();
if (_reindexJobRecord.Count == 0)
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Information,
OperationOutcomeConstants.IssueType.Informational,
Core.Resources.NoResourcesNeedToBeReindexed));
await UpdateParametersAndCompleteJob();
return(false);
}
// Generate separate queries for each resource type and add them to query list.
foreach (string resourceType in _reindexJobRecord.Resources)
{
// Checking resource specific counts is a performance improvement,
// so if an entry for this resource failed to get added to the count dictionary, run a query anyways
if (!_reindexJobRecord.ResourceCounts.ContainsKey(resourceType) || _reindexJobRecord.ResourceCounts[resourceType] > 0)
{
var query = new ReindexJobQueryStatus(resourceType, continuationToken: null)
{
LastModified = Clock.UtcNow,
Status = OperationStatus.Queued,
};
_reindexJobRecord.QueryList.TryAdd(query, 1);
}
}
await UpdateJobAsync();
_throttleController.UpdateDatastoreUsage();
return(true);
}
private async Task <bool> TryPopulateNewJobFields()
{
// Build query based on new search params
// Find supported, but not yet searchable params
var notYetIndexedParams = _supportedSearchParameterDefinitionManager.GetSearchParametersRequiringReindexing();
// if there are not any parameters which are supported but not yet indexed, then we have nothing to do
if (!notYetIndexedParams.Any())
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Information,
OperationOutcomeConstants.IssueType.Informational,
Resources.NoSearchParametersNeededToBeIndexed));
_reindexJobRecord.CanceledTime = Clock.UtcNow;
await MoveToFinalStatusAsync(OperationStatus.Canceled);
return(false);
}
// From the param list, get the list of necessary resources which should be
// included in our query
var resourceList = new HashSet <string>();
foreach (var param in notYetIndexedParams)
{
foreach (var baseResourceType in param.BaseResourceTypes)
{
if (baseResourceType == KnownResourceTypes.Resource)
{
resourceList.UnionWith(_modelInfoProvider.GetResourceTypeNames().ToHashSet());
// We added all possible resource types, so no need to continue
break;
}
if (baseResourceType == KnownResourceTypes.DomainResource)
{
var domainResourceChildResourceTypes = _modelInfoProvider.GetResourceTypeNames().ToHashSet();
// Remove types that inherit from Resource directly
domainResourceChildResourceTypes.Remove(KnownResourceTypes.Binary);
domainResourceChildResourceTypes.Remove(KnownResourceTypes.Bundle);
domainResourceChildResourceTypes.Remove(KnownResourceTypes.Parameters);
resourceList.UnionWith(domainResourceChildResourceTypes);
}
else
{
resourceList.UnionWith(new[] { baseResourceType });
}
}
}
foreach (var resource in resourceList)
{
_reindexJobRecord.Resources.Add(resource);
}
foreach (var searchParams in notYetIndexedParams.Select(p => p.Url.ToString()))
{
_reindexJobRecord.SearchParams.Add(searchParams);
}
await CalculateTotalAndResourceCounts();
if (_reindexJobRecord.Count == 0)
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Information,
OperationOutcomeConstants.IssueType.Informational,
Resources.NoResourcesNeedToBeReindexed));
await UpdateParametersAndCompleteJob();
return(false);
}
// Generate separate queries for each resource type and add them to query list.
foreach (string resourceType in _reindexJobRecord.Resources)
{
// Checking resource specific counts is a performance improvement,
// so if an entry for this resource failed to get added to the count dictionary, run a query anyways
if (!_reindexJobRecord.ResourceCounts.ContainsKey(resourceType) || _reindexJobRecord.ResourceCounts[resourceType] > 0)
{
var query = new ReindexJobQueryStatus(resourceType, continuationToken: null)
{
LastModified = Clock.UtcNow,
Status = OperationStatus.Queued,
};
_reindexJobRecord.QueryList.TryAdd(query, 1);
}
}
await UpdateJobAsync();
_throttleController.UpdateDatastoreUsage();
return(true);
}
/// <inheritdoc />
public async Task ExecuteAsync(ReindexJobRecord reindexJobRecord, WeakETag weakETag, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(reindexJobRecord, nameof(reindexJobRecord));
EnsureArg.IsNotNull(weakETag, nameof(weakETag));
_reindexJobRecord = reindexJobRecord;
_weakETag = weakETag;
try
{
if (_reindexJobRecord.Status != OperationStatus.Running)
{
// update job record to running
_reindexJobRecord.Status = OperationStatus.Running;
_reindexJobRecord.StartTime = Clock.UtcNow;
await UpdateJobAsync(cancellationToken);
}
// If we are resuming a job, we can detect that by checking the progress info from the job record.
// If no queries have been added to the progress then this is a new job
if (_reindexJobRecord.QueryList?.Count == 0)
{
// Build query based on new search params
// Find supported, but not yet searchable params
var notYetIndexedParams = _supportedSearchParameterDefinitionManager.GetSupportedButNotSearchableParams();
// if there are not any parameters which are supported but not yet indexed, then we have nothing to do
if (!notYetIndexedParams.Any())
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Information,
OperationOutcomeConstants.IssueType.Informational,
Resources.NoSearchParametersNeededToBeIndexed));
_reindexJobRecord.CanceledTime = DateTimeOffset.UtcNow;
await CompleteJobAsync(OperationStatus.Canceled, cancellationToken);
return;
}
// From the param list, get the list of necessary resources which should be
// included in our query
var resourceList = new HashSet <string>();
foreach (var param in notYetIndexedParams)
{
resourceList.UnionWith(param.TargetResourceTypes);
// TODO: Expand the BaseResourceTypes to all child resources
resourceList.UnionWith(param.BaseResourceTypes);
}
_reindexJobRecord.Resources.AddRange(resourceList);
_reindexJobRecord.SearchParams.AddRange(notYetIndexedParams.Select(p => p.Name));
// generate and run first query
var queryStatus = new ReindexJobQueryStatus(null);
queryStatus.LastModified = DateTimeOffset.UtcNow;
queryStatus.Status = OperationStatus.Queued;
_reindexJobRecord.QueryList.Add(queryStatus);
// update the complete total
var countOnlyResults = await ExecuteReindexQueryAsync(queryStatus, countOnly : true, cancellationToken);
_reindexJobRecord.Count = countOnlyResults.TotalCount.Value;
// Query first batch of resources
await ProcessQueryAsync(queryStatus, cancellationToken);
}
else
{
// check to see if queries are queued
// TODO: this while loop is temporary until we multithread this task so multiple threads can
// processes queries
while (_reindexJobRecord.QueryList.Where(q => q.Status == OperationStatus.Queued).Any())
{
// grab the next query from the list which is labeled as queued and run it
var query = _reindexJobRecord.QueryList.Where(q => q.Status == OperationStatus.Queued).OrderBy(q => q.LastModified).FirstOrDefault();
await ProcessQueryAsync(query, cancellationToken);
}
}
}
catch (JobConflictException)
{
// The reindex job was updated externally.
_logger.LogTrace("The job was updated by another process.");
}
catch (Exception ex)
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Error,
OperationOutcomeConstants.IssueType.Exception,
ex.Message));
_reindexJobRecord.FailureCount++;
_logger.LogError(ex, $"Encountered an unhandled exception. The job failure count increased to {_reindexJobRecord.FailureCount}.");
await UpdateJobAsync(cancellationToken);
if (_reindexJobRecord.FailureCount >= _reindexJobConfiguration.ConsecutiveFailuresThreshold)
{
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
}
}
