public async Task InitializeAsync()
{
_fhirOperationDataStore = _fixture.OperationDataStore;
_fhirStorageTestHelper = _fixture.TestHelper;
_scopedOperationDataStore = _fhirOperationDataStore.CreateMockScope();
_scopedDataStore = _fixture.DataStore.CreateMockScope();
_jobConfiguration = new ReindexJobConfiguration();
IOptions <ReindexJobConfiguration> optionsReindexConfig = Substitute.For <IOptions <ReindexJobConfiguration> >();
optionsReindexConfig.Value.Returns(_jobConfiguration);
_searchParameterDefinitionManager = _fixture.SearchParameterDefinitionManager;
_supportedSearchParameterDefinitionManager = _fixture.SupportedSearchParameterDefinitionManager;
ResourceWrapperFactory wrapperFactory = Mock.TypeWithArguments <ResourceWrapperFactory>(
new RawResourceFactory(new FhirJsonSerializer()),
new FhirRequestContextAccessor(),
_searchIndexer,
_searchParameterDefinitionManager,
Deserializers.ResourceDeserializer);
_searchParameterStatusManager = _fixture.SearchParameterStatusManager;
_createReindexRequestHandler = new CreateReindexRequestHandler(
_fhirOperationDataStore,
DisabledFhirAuthorizationService.Instance,
optionsReindexConfig,
_searchParameterDefinitionManager,
_searchParameterOperations);
_reindexUtilities = new ReindexUtilities(
() => _scopedDataStore,
_searchIndexer,
Deserializers.ResourceDeserializer,
_supportedSearchParameterDefinitionManager,
_searchParameterStatusManager,
wrapperFactory);
_searchService = _fixture.SearchService.CreateMockScope();
await _fhirStorageTestHelper.DeleteAllReindexJobRecordsAsync(CancellationToken.None);
_throttleController.GetThrottleBasedDelay().Returns(0);
_throttleController.GetThrottleBatchSize().Returns(100U);
}
/// <inheritdoc />
public async Task ExecuteAsync(ReindexJobRecord reindexJobRecord, WeakETag weakETag, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(reindexJobRecord, nameof(reindexJobRecord));
EnsureArg.IsNotNull(weakETag, nameof(weakETag));
_reindexJobRecord = reindexJobRecord;
_weakETag = weakETag;
var jobSemaphore = new SemaphoreSlim(1, 1);
var existingFhirRequestContext = _contextAccessor.FhirRequestContext;
try
{
// Add a request context so Datastore consumption can be added
var fhirRequestContext = new FhirRequestContext(
method: OperationsConstants.Reindex,
uriString: "$reindex",
baseUriString: "$reindex",
correlationId: _reindexJobRecord.Id,
requestHeaders: new Dictionary <string, StringValues>(),
responseHeaders: new Dictionary <string, StringValues>())
{
IsBackgroundTask = true,
AuditEventType = OperationsConstants.Reindex,
};
_contextAccessor.FhirRequestContext = fhirRequestContext;
using (IScoped <IFhirDataStore> store = _fhirDataStoreFactory())
{
var provisionedCapacity = await store.Value.GetProvisionedDataStoreCapacityAsync(cancellationToken);
_throttleController.Initialize(_reindexJobRecord, provisionedCapacity);
}
if (_reindexJobRecord.Status != OperationStatus.Running ||
_reindexJobRecord.StartTime == null)
{
// update job record to running
_reindexJobRecord.Status = OperationStatus.Running;
_reindexJobRecord.StartTime = Clock.UtcNow;
await UpdateJobAsync(cancellationToken);
}
// If we are resuming a job, we can detect that by checking the progress info from the job record.
// If no queries have been added to the progress then this is a new job
if (_reindexJobRecord.QueryList?.Count == 0)
{
// Build query based on new search params
// Find supported, but not yet searchable params
var notYetIndexedParams = _supportedSearchParameterDefinitionManager.GetSearchParametersRequiringReindexing();
// if there are not any parameters which are supported but not yet indexed, then we have nothing to do
if (!notYetIndexedParams.Any())
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Information,
OperationOutcomeConstants.IssueType.Informational,
Resources.NoSearchParametersNeededToBeIndexed));
_reindexJobRecord.CanceledTime = DateTimeOffset.UtcNow;
await CompleteJobAsync(OperationStatus.Canceled, cancellationToken);
return;
}
// From the param list, get the list of necessary resources which should be
// included in our query
var resourceList = new HashSet <string>();
foreach (var param in notYetIndexedParams)
{
foreach (var baseResourceType in param.BaseResourceTypes)
{
if (baseResourceType == KnownResourceTypes.Resource)
{
resourceList.UnionWith(_modelInfoProvider.GetResourceTypeNames().ToHashSet());
// We added all possible resource types, so no need to continue
break;
}
if (baseResourceType == KnownResourceTypes.DomainResource)
{
var domainResourceChildResourceTypes = _modelInfoProvider.GetResourceTypeNames().ToHashSet();
// Remove types that inherit from Resource directly
domainResourceChildResourceTypes.Remove(KnownResourceTypes.Binary);
domainResourceChildResourceTypes.Remove(KnownResourceTypes.Bundle);
domainResourceChildResourceTypes.Remove(KnownResourceTypes.Parameters);
resourceList.UnionWith(domainResourceChildResourceTypes);
}
else
{
resourceList.UnionWith(new[] { baseResourceType });
}
}
}
_reindexJobRecord.Resources.AddRange(resourceList);
_reindexJobRecord.SearchParams.AddRange(notYetIndexedParams.Select(p => p.Url.ToString()));
await CalculateTotalAndResourceCounts(cancellationToken);
if (_reindexJobRecord.Count == 0)
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Information,
OperationOutcomeConstants.IssueType.Informational,
Resources.NoResourcesNeedToBeReindexed));
await UpdateParametersAndCompleteJob(cancellationToken);
return;
}
// Generate separate queries for each resource type and add them to query list.
foreach (string resourceType in _reindexJobRecord.Resources)
{
// Checking resource specific counts is a performance improvement,
// so if an entry for this resource failed to get added to the count dictionary, run a query anyways
if (!_reindexJobRecord.ResourceCounts.ContainsKey(resourceType) || _reindexJobRecord.ResourceCounts[resourceType] > 0)
{
var query = new ReindexJobQueryStatus(resourceType, continuationToken: null)
{
LastModified = Clock.UtcNow,
Status = OperationStatus.Queued,
};
_reindexJobRecord.QueryList.TryAdd(query, 1);
}
}
await UpdateJobAsync(cancellationToken);
_throttleController.UpdateDatastoreUsage();
}
var queryTasks = new List <Task <ReindexJobQueryStatus> >();
var queryCancellationTokens = new Dictionary <ReindexJobQueryStatus, CancellationTokenSource>();
// while not all queries are finished
while (_reindexJobRecord.QueryList.Keys.Where(q =>
q.Status == OperationStatus.Queued ||
q.Status == OperationStatus.Running).Any())
{
if (_reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Queued).Any())
{
// grab the next query from the list which is labeled as queued and run it
var query = _reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Queued).OrderBy(q => q.LastModified).FirstOrDefault();
CancellationTokenSource queryTokensSource = new CancellationTokenSource();
queryCancellationTokens.TryAdd(query, queryTokensSource);
#pragma warning disable CS4014 // Suppressed as we want to continue execution and begin processing the next query while this continues to run
queryTasks.Add(ProcessQueryAsync(query, jobSemaphore, queryTokensSource.Token));
#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
_logger.LogInformation($"Reindex job task created {queryTasks.Count} Tasks");
}
// reset stale queries to pending
var staleQueries = _reindexJobRecord.QueryList.Keys.Where(
q => q.Status == OperationStatus.Running && q.LastModified < Clock.UtcNow - _reindexJobConfiguration.JobHeartbeatTimeoutThreshold);
foreach (var staleQuery in staleQueries)
{
await jobSemaphore.WaitAsync();
try
{
// if this query has a created task, cancel it
if (queryCancellationTokens.TryGetValue(staleQuery, out var tokenSource))
{
try
{
tokenSource.Cancel(false);
}
catch
{
// may throw exception if the task is disposed
}
}
staleQuery.Status = OperationStatus.Queued;
await UpdateJobAsync(cancellationToken);
}
finally
{
jobSemaphore.Release();
}
}
var averageDbConsumption = _throttleController.UpdateDatastoreUsage();
_logger.LogInformation($"Reindex avaerage DB consumption: {averageDbConsumption}");
var throttleDelayTime = _throttleController.GetThrottleBasedDelay();
_logger.LogInformation($"Reindex throttle delay: {throttleDelayTime}");
await Task.Delay(_reindexJobRecord.QueryDelayIntervalInMilliseconds + throttleDelayTime);
// Remove all finished tasks from the collections of tasks
// and cancellationTokens
if (queryTasks.Count >= reindexJobRecord.MaximumConcurrency)
{
var taskArray = queryTasks.ToArray();
Task.WaitAny(taskArray);
var finishedTasks = queryTasks.Where(t => t.IsCompleted).ToArray();
foreach (var finishedTask in finishedTasks)
{
queryTasks.Remove(finishedTask);
queryCancellationTokens.Remove(await finishedTask);
}
}
// if our received CancellationToken is cancelled we should
// pass that cancellation request onto all the cancellationTokens
// for the currently executing threads
if (cancellationToken.IsCancellationRequested)
{
foreach (var tokenSource in queryCancellationTokens.Values)
{
tokenSource.Cancel(false);
}
}
}
Task.WaitAll(queryTasks.ToArray());
await jobSemaphore.WaitAsync();
try
{
await CheckJobCompletionStatus(cancellationToken);
}
finally
{
jobSemaphore.Release();
}
}
catch (JobConflictException)
{
// The reindex job was updated externally.
_logger.LogInformation("The job was updated by another process.");
}
catch (Exception ex)
{
await jobSemaphore.WaitAsync();
try
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Error,
OperationOutcomeConstants.IssueType.Exception,
ex.Message));
_reindexJobRecord.FailureCount++;
_logger.LogError(ex, "Encountered an unhandled exception. The job failure count increased to {failureCount}.", _reindexJobRecord.FailureCount);
await UpdateJobAsync(cancellationToken);
if (_reindexJobRecord.FailureCount >= _reindexJobConfiguration.ConsecutiveFailuresThreshold)
{
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
else
{
_reindexJobRecord.Status = OperationStatus.Queued;
await UpdateJobAsync(cancellationToken);
}
}
finally
{
jobSemaphore.Release();
}
}
finally
{
jobSemaphore.Dispose();
_contextAccessor.FhirRequestContext = existingFhirRequestContext;
}
}
private async Task ProcessJob()
{
var queryTasks = new List <Task <ReindexJobQueryStatus> >();
var queryCancellationTokens = new Dictionary <ReindexJobQueryStatus, CancellationTokenSource>();
// while not all queries are finished
while (_reindexJobRecord.QueryList.Keys.Where(q =>
q.Status == OperationStatus.Queued ||
q.Status == OperationStatus.Running).Any())
{
if (_reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Queued).Any())
{
// grab the next query from the list which is labeled as queued and run it
var query = _reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Queued).OrderBy(q => q.LastModified).FirstOrDefault();
CancellationTokenSource queryTokensSource = new CancellationTokenSource();
queryCancellationTokens.TryAdd(query, queryTokensSource);
// We don't await ProcessQuery, so query status can or can not be changed inside immediately
// In some cases we can go th6rough whole loop and pick same query from query list.
// To prevent that we marking query as running here and not inside ProcessQuery code.
query.Status = OperationStatus.Running;
query.LastModified = Clock.UtcNow;
#pragma warning disable CS4014 // Suppressed as we want to continue execution and begin processing the next query while this continues to run
queryTasks.Add(ProcessQueryAsync(query, queryTokensSource.Token));
#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
_logger.LogInformation($"Reindex job task created {queryTasks.Count} Tasks");
}
// reset stale queries to pending
var staleQueries = _reindexJobRecord.QueryList.Keys.Where(
q => q.Status == OperationStatus.Running && q.LastModified < Clock.UtcNow - _reindexJobConfiguration.JobHeartbeatTimeoutThreshold);
foreach (var staleQuery in staleQueries)
{
await _jobSemaphore.WaitAsync(_cancellationToken);
try
{
// if this query has a created task, cancel it
if (queryCancellationTokens.TryGetValue(staleQuery, out var tokenSource))
{
try
{
tokenSource.Cancel(false);
}
catch
{
// may throw exception if the task is disposed
}
}
staleQuery.Status = OperationStatus.Queued;
await UpdateJobAsync();
}
finally
{
_jobSemaphore.Release();
}
}
var averageDbConsumption = _throttleController.UpdateDatastoreUsage();
_logger.LogInformation($"Reindex avaerage DB consumption: {averageDbConsumption}");
var throttleDelayTime = _throttleController.GetThrottleBasedDelay();
_logger.LogInformation($"Reindex throttle delay: {throttleDelayTime}");
await Task.Delay(_reindexJobRecord.QueryDelayIntervalInMilliseconds + throttleDelayTime, _cancellationToken);
// Remove all finished tasks from the collections of tasks
// and cancellationTokens
if (queryTasks.Count >= _reindexJobRecord.MaximumConcurrency)
{
var taskArray = queryTasks.ToArray();
Task.WaitAny(taskArray, _cancellationToken);
var finishedTasks = queryTasks.Where(t => t.IsCompleted).ToArray();
foreach (var finishedTask in finishedTasks)
{
queryTasks.Remove(finishedTask);
queryCancellationTokens.Remove(await finishedTask);
}
}
// for most cases if another process updates the job (such as a DELETE request)
// the _etag change will cause a JobConflict exception and this task will be aborted
// but here we add one more check before attempting to mark the job as complete,
// or starting another iteration of the loop
await _jobSemaphore.WaitAsync();
try
{
using (IScoped <IFhirOperationDataStore> store = _fhirOperationDataStoreFactory.Invoke())
{
var wrapper = await store.Value.GetReindexJobByIdAsync(_reindexJobRecord.Id, _cancellationToken);
_weakETag = wrapper.ETag;
_reindexJobRecord.Status = wrapper.JobRecord.Status;
}
}
catch (Exception)
{
// if something went wrong with fetching job status, we shouldn't fail process loop.
}
finally
{
_jobSemaphore.Release();
}
// if our received CancellationToken is cancelled, or the job has been marked canceled we should
// pass that cancellation request onto all the cancellationTokens
// for the currently executing threads
if (_cancellationToken.IsCancellationRequested || _reindexJobRecord.Status == OperationStatus.Canceled)
{
foreach (var tokenSource in queryCancellationTokens.Values)
{
tokenSource.Cancel(false);
}
_logger.LogInformation("Reindex Job canceled.");
throw new OperationCanceledException("ReindexJob canceled.");
}
}
Task.WaitAll(queryTasks.ToArray(), _cancellationToken);
}