private ICollection <string> GetDerivedResourceTypes(IReadOnlyCollection <string> resourceTypes)
{
var completeResourceList = new HashSet <string>(resourceTypes);
foreach (var baseResourceType in resourceTypes)
{
if (baseResourceType == KnownResourceTypes.Resource)
{
completeResourceList.UnionWith(_modelInfoProvider.GetResourceTypeNames().ToHashSet());
// We added all possible resource types, so no need to continue
break;
}
if (baseResourceType == KnownResourceTypes.DomainResource)
{
var domainResourceChildResourceTypes = _modelInfoProvider.GetResourceTypeNames().ToHashSet();
// Remove types that inherit from Resource directly
domainResourceChildResourceTypes.Remove(KnownResourceTypes.Binary);
domainResourceChildResourceTypes.Remove(KnownResourceTypes.Bundle);
domainResourceChildResourceTypes.Remove(KnownResourceTypes.Parameters);
completeResourceList.UnionWith(domainResourceChildResourceTypes);
}
}
return(completeResourceList);
}
public static MockModelInfoProviderBuilder Create(FhirSpecification version)
{
IModelInfoProvider provider = Substitute.For <IModelInfoProvider>();
provider.Version.Returns(version);
// Adds normative types by default
var seenTypes = new HashSet <string>
{
"Binary", "Bundle", "CapabilityStatement", "CodeSystem", "Observation", "OperationOutcome", "Patient", "StructureDefinition", "ValueSet",
};
provider.GetResourceTypeNames().Returns(_ => seenTypes.Where(x => !string.IsNullOrEmpty(x)).ToArray());
provider.IsKnownResource(Arg.Any <string>()).Returns(x => provider.GetResourceTypeNames().Contains(x[0]));
// Simulate inherited behavior
// Some code depends on "InheritedResource".BaseType
// This adds the ability to resolve "Resource" as the base type
provider.GetTypeForFhirType(Arg.Any <string>()).Returns(p => p.ArgAt <string>(0) == "Resource" ? typeof(ResourceObj) : typeof(InheritedResourceObj));
provider.GetFhirTypeNameForType(Arg.Any <Type>()).Returns(p => p.ArgAt <Type>(0) == typeof(ResourceObj) ? "Resource" : null);
// IStructureDefinitionSummaryProvider allows the execution of FHIRPath queries
provider.ToTypedElement(Arg.Any <ISourceNode>())
.Returns(p => p.ArgAt <ISourceNode>(0).ToTypedElement(new MockStructureDefinitionSummaryProvider(p.ArgAt <ISourceNode>(0), seenTypes)));
return(new MockModelInfoProviderBuilder(provider, seenTypes));
}
public ICapabilityStatementBuilder PopulateDefaultResourceInteractions()
{
foreach (string resource in _modelInfoProvider.GetResourceTypeNames())
{
// Parameters is a non-persisted resource used to pass information into and back from an operation.
if (string.Equals(resource, KnownResourceTypes.Parameters, StringComparison.Ordinal))
{
continue;
}
AddResourceInteraction(resource, TypeRestfulInteraction.Create);
AddResourceInteraction(resource, TypeRestfulInteraction.Read);
AddResourceInteraction(resource, TypeRestfulInteraction.Vread);
AddResourceInteraction(resource, TypeRestfulInteraction.HistoryType);
AddResourceInteraction(resource, TypeRestfulInteraction.HistoryInstance);
// AuditEvents should not allow Update or Delete
if (!string.Equals(resource, KnownResourceTypes.AuditEvent, StringComparison.Ordinal))
{
AddResourceInteraction(resource, TypeRestfulInteraction.Update);
AddResourceInteraction(resource, TypeRestfulInteraction.Patch);
AddResourceInteraction(resource, TypeRestfulInteraction.Delete);
}
ApplyToResource(resource, component =>
{
component.Versioning.Add(ResourceVersionPolicy.NoVersion);
component.Versioning.Add(ResourceVersionPolicy.Versioned);
component.Versioning.Add(ResourceVersionPolicy.VersionedUpdate);
// Create is added for every resource above.
component.ConditionalCreate = true;
// AuditEvent don't allow update, so no conditional update as well.
if (!string.Equals(resource, KnownResourceTypes.AuditEvent, StringComparison.Ordinal))
{
component.ConditionalUpdate = true;
component.ConditionalDelete.Add(ConditionalDeleteStatus.NotSupported);
component.ConditionalDelete.Add(ConditionalDeleteStatus.Single);
component.ConditionalDelete.Add(ConditionalDeleteStatus.Multiple);
}
component.ReadHistory = true;
component.UpdateCreate = true;
});
}
AddGlobalInteraction(SystemRestfulInteraction.HistorySystem);
return(this);
}
internal void Build()
{
ILookup <string, SearchParameterInfo> searchParametersLookup = ValidateAndGetFlattenedList()
.ToLookup(
entry => entry.ResourceType,
entry => entry.SearchParameter);
// Build the inheritance. For example, the _id search parameter is on Resource
// and should be available to all resources that inherit Resource.
foreach (string resourceType in _modelInfoProvider.GetResourceTypeNames())
{
if (_resourceTypeDictionary.TryGetValue(resourceType, out IDictionary <string, SearchParameterInfo> _))
{
// The list has already been built, move on.
continue;
}
// Recursively build the search parameter definitions. For example,
// Appointment inherits from DomainResource, which inherits from Resource
// and therefore Appointment should include all search parameters DomainResource and Resource supports.
BuildSearchParameterDefinition(searchParametersLookup, resourceType);
}
_initialized = true;
}
internal static void Build(
IReadOnlyCollection <ITypedElement> searchParameters,
IDictionary <Uri, SearchParameterInfo> uriDictionary,
IDictionary <string, IDictionary <string, SearchParameterInfo> > resourceTypeDictionary,
IModelInfoProvider modelInfoProvider)
{
EnsureArg.IsNotNull(searchParameters, nameof(searchParameters));
EnsureArg.IsNotNull(uriDictionary, nameof(uriDictionary));
EnsureArg.IsNotNull(resourceTypeDictionary, nameof(resourceTypeDictionary));
EnsureArg.IsNotNull(modelInfoProvider, nameof(modelInfoProvider));
ILookup <string, SearchParameterInfo> searchParametersLookup = ValidateAndGetFlattenedList(
searchParameters,
uriDictionary,
modelInfoProvider).ToLookup(
entry => entry.ResourceType,
entry => entry.SearchParameter);
// Build the inheritance. For example, the _id search parameter is on Resource
// and should be available to all resources that inherit Resource.
foreach (string resourceType in modelInfoProvider.GetResourceTypeNames())
{
// Recursively build the search parameter definitions. For example,
// Appointment inherits from DomainResource, which inherits from Resource
// and therefore Appointment should include all search parameters DomainResource and Resource supports.
BuildSearchParameterDefinition(searchParametersLookup, resourceType, resourceTypeDictionary, modelInfoProvider);
}
}
public void Build(IListedCapabilityStatement statement)
{
foreach (var resource in _modelInfoProvider.GetResourceTypeNames())
{
statement.TryAddRestInteraction(resource, TypeRestfulInteraction.HistoryType);
statement.TryAddRestInteraction(resource, TypeRestfulInteraction.HistoryInstance);
}
statement.TryAddRestInteraction(SystemRestfulInteraction.HistorySystem);
}
public ICapabilityStatementBuilder AddDefaultResourceInteractions()
{
foreach (string resource in _modelInfoProvider.GetResourceTypeNames())
{
// Parameters is a non-persisted resource used to pass information into and back from an operation.
if (string.Equals(resource, KnownResourceTypes.Parameters, StringComparison.Ordinal))
{
continue;
}
AddRestInteraction(resource, TypeRestfulInteraction.Create);
AddRestInteraction(resource, TypeRestfulInteraction.Read);
AddRestInteraction(resource, TypeRestfulInteraction.Vread);
AddRestInteraction(resource, TypeRestfulInteraction.HistoryType);
AddRestInteraction(resource, TypeRestfulInteraction.HistoryInstance);
// AuditEvents should not allow Update or Delete
if (!string.Equals(resource, KnownResourceTypes.AuditEvent, StringComparison.Ordinal))
{
AddRestInteraction(resource, TypeRestfulInteraction.Update);
AddRestInteraction(resource, TypeRestfulInteraction.Delete);
}
UpdateRestResourceComponent(resource, component =>
{
component.Versioning.Add(ResourceVersionPolicy.NoVersion);
component.Versioning.Add(ResourceVersionPolicy.Versioned);
component.Versioning.Add(ResourceVersionPolicy.VersionedUpdate);
component.ReadHistory = true;
component.UpdateCreate = true;
});
}
AddRestInteraction(SystemRestfulInteraction.HistorySystem);
return(this);
}
public void Build(IListedCapabilityStatement statement)
{
EnsureArg.IsNotNull(statement, nameof(statement));
foreach (var resource in _modelInfoProvider.GetResourceTypeNames())
{
statement.BuildRestResourceComponent(resource, builder =>
{
builder.AddResourceVersionPolicy(ResourceVersionPolicy.NoVersion);
builder.AddResourceVersionPolicy(ResourceVersionPolicy.Versioned);
builder.AddResourceVersionPolicy(ResourceVersionPolicy.VersionedUpdate);
builder.ReadHistory = true;
builder.UpdateCreate = true;
});
}
}
/// <inheritdoc />
public async Task ExecuteAsync(ReindexJobRecord reindexJobRecord, WeakETag weakETag, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(reindexJobRecord, nameof(reindexJobRecord));
EnsureArg.IsNotNull(weakETag, nameof(weakETag));
_reindexJobRecord = reindexJobRecord;
_weakETag = weakETag;
var jobSemaphore = new SemaphoreSlim(1, 1);
var existingFhirRequestContext = _contextAccessor.FhirRequestContext;
try
{
// Add a request context so Datastore consumption can be added
var fhirRequestContext = new FhirRequestContext(
method: OperationsConstants.Reindex,
uriString: "$reindex",
baseUriString: "$reindex",
correlationId: _reindexJobRecord.Id,
requestHeaders: new Dictionary <string, StringValues>(),
responseHeaders: new Dictionary <string, StringValues>())
{
IsBackgroundTask = true,
AuditEventType = OperationsConstants.Reindex,
};
_contextAccessor.FhirRequestContext = fhirRequestContext;
using (IScoped <IFhirDataStore> store = _fhirDataStoreFactory())
{
var provisionedCapacity = await store.Value.GetProvisionedDataStoreCapacityAsync(cancellationToken);
_throttleController.Initialize(_reindexJobRecord, provisionedCapacity);
}
if (_reindexJobRecord.Status != OperationStatus.Running ||
_reindexJobRecord.StartTime == null)
{
// update job record to running
_reindexJobRecord.Status = OperationStatus.Running;
_reindexJobRecord.StartTime = Clock.UtcNow;
await UpdateJobAsync(cancellationToken);
}
// If we are resuming a job, we can detect that by checking the progress info from the job record.
// If no queries have been added to the progress then this is a new job
if (_reindexJobRecord.QueryList?.Count == 0)
{
// Build query based on new search params
// Find supported, but not yet searchable params
var notYetIndexedParams = _supportedSearchParameterDefinitionManager.GetSearchParametersRequiringReindexing();
// if there are not any parameters which are supported but not yet indexed, then we have nothing to do
if (!notYetIndexedParams.Any())
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Information,
OperationOutcomeConstants.IssueType.Informational,
Resources.NoSearchParametersNeededToBeIndexed));
_reindexJobRecord.CanceledTime = DateTimeOffset.UtcNow;
await CompleteJobAsync(OperationStatus.Canceled, cancellationToken);
return;
}
// From the param list, get the list of necessary resources which should be
// included in our query
var resourceList = new HashSet <string>();
foreach (var param in notYetIndexedParams)
{
foreach (var baseResourceType in param.BaseResourceTypes)
{
if (baseResourceType == KnownResourceTypes.Resource)
{
resourceList.UnionWith(_modelInfoProvider.GetResourceTypeNames().ToHashSet());
// We added all possible resource types, so no need to continue
break;
}
if (baseResourceType == KnownResourceTypes.DomainResource)
{
var domainResourceChildResourceTypes = _modelInfoProvider.GetResourceTypeNames().ToHashSet();
// Remove types that inherit from Resource directly
domainResourceChildResourceTypes.Remove(KnownResourceTypes.Binary);
domainResourceChildResourceTypes.Remove(KnownResourceTypes.Bundle);
domainResourceChildResourceTypes.Remove(KnownResourceTypes.Parameters);
resourceList.UnionWith(domainResourceChildResourceTypes);
}
else
{
resourceList.UnionWith(new[] { baseResourceType });
}
}
}
_reindexJobRecord.Resources.AddRange(resourceList);
_reindexJobRecord.SearchParams.AddRange(notYetIndexedParams.Select(p => p.Url.ToString()));
await CalculateTotalAndResourceCounts(cancellationToken);
if (_reindexJobRecord.Count == 0)
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Information,
OperationOutcomeConstants.IssueType.Informational,
Resources.NoResourcesNeedToBeReindexed));
await UpdateParametersAndCompleteJob(cancellationToken);
return;
}
// Generate separate queries for each resource type and add them to query list.
foreach (string resourceType in _reindexJobRecord.Resources)
{
// Checking resource specific counts is a performance improvement,
// so if an entry for this resource failed to get added to the count dictionary, run a query anyways
if (!_reindexJobRecord.ResourceCounts.ContainsKey(resourceType) || _reindexJobRecord.ResourceCounts[resourceType] > 0)
{
var query = new ReindexJobQueryStatus(resourceType, continuationToken: null)
{
LastModified = Clock.UtcNow,
Status = OperationStatus.Queued,
};
_reindexJobRecord.QueryList.TryAdd(query, 1);
}
}
await UpdateJobAsync(cancellationToken);
_throttleController.UpdateDatastoreUsage();
}
var queryTasks = new List <Task <ReindexJobQueryStatus> >();
var queryCancellationTokens = new Dictionary <ReindexJobQueryStatus, CancellationTokenSource>();
// while not all queries are finished
while (_reindexJobRecord.QueryList.Keys.Where(q =>
q.Status == OperationStatus.Queued ||
q.Status == OperationStatus.Running).Any())
{
if (_reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Queued).Any())
{
// grab the next query from the list which is labeled as queued and run it
var query = _reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Queued).OrderBy(q => q.LastModified).FirstOrDefault();
CancellationTokenSource queryTokensSource = new CancellationTokenSource();
queryCancellationTokens.TryAdd(query, queryTokensSource);
#pragma warning disable CS4014 // Suppressed as we want to continue execution and begin processing the next query while this continues to run
queryTasks.Add(ProcessQueryAsync(query, jobSemaphore, queryTokensSource.Token));
#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
_logger.LogInformation($"Reindex job task created {queryTasks.Count} Tasks");
}
// reset stale queries to pending
var staleQueries = _reindexJobRecord.QueryList.Keys.Where(
q => q.Status == OperationStatus.Running && q.LastModified < Clock.UtcNow - _reindexJobConfiguration.JobHeartbeatTimeoutThreshold);
foreach (var staleQuery in staleQueries)
{
await jobSemaphore.WaitAsync();
try
{
// if this query has a created task, cancel it
if (queryCancellationTokens.TryGetValue(staleQuery, out var tokenSource))
{
try
{
tokenSource.Cancel(false);
}
catch
{
// may throw exception if the task is disposed
}
}
staleQuery.Status = OperationStatus.Queued;
await UpdateJobAsync(cancellationToken);
}
finally
{
jobSemaphore.Release();
}
}
var averageDbConsumption = _throttleController.UpdateDatastoreUsage();
_logger.LogInformation($"Reindex avaerage DB consumption: {averageDbConsumption}");
var throttleDelayTime = _throttleController.GetThrottleBasedDelay();
_logger.LogInformation($"Reindex throttle delay: {throttleDelayTime}");
await Task.Delay(_reindexJobRecord.QueryDelayIntervalInMilliseconds + throttleDelayTime);
// Remove all finished tasks from the collections of tasks
// and cancellationTokens
if (queryTasks.Count >= reindexJobRecord.MaximumConcurrency)
{
var taskArray = queryTasks.ToArray();
Task.WaitAny(taskArray);
var finishedTasks = queryTasks.Where(t => t.IsCompleted).ToArray();
foreach (var finishedTask in finishedTasks)
{
queryTasks.Remove(finishedTask);
queryCancellationTokens.Remove(await finishedTask);
}
}
// if our received CancellationToken is cancelled we should
// pass that cancellation request onto all the cancellationTokens
// for the currently executing threads
if (cancellationToken.IsCancellationRequested)
{
foreach (var tokenSource in queryCancellationTokens.Values)
{
tokenSource.Cancel(false);
}
}
}
Task.WaitAll(queryTasks.ToArray());
await jobSemaphore.WaitAsync();
try
{
await CheckJobCompletionStatus(cancellationToken);
}
finally
{
jobSemaphore.Release();
}
}
catch (JobConflictException)
{
// The reindex job was updated externally.
_logger.LogInformation("The job was updated by another process.");
}
catch (Exception ex)
{
await jobSemaphore.WaitAsync();
try
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Error,
OperationOutcomeConstants.IssueType.Exception,
ex.Message));
_reindexJobRecord.FailureCount++;
_logger.LogError(ex, "Encountered an unhandled exception. The job failure count increased to {failureCount}.", _reindexJobRecord.FailureCount);
await UpdateJobAsync(cancellationToken);
if (_reindexJobRecord.FailureCount >= _reindexJobConfiguration.ConsecutiveFailuresThreshold)
{
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
else
{
_reindexJobRecord.Status = OperationStatus.Queued;
await UpdateJobAsync(cancellationToken);
}
}
finally
{
jobSemaphore.Release();
}
}
finally
{
jobSemaphore.Dispose();
_contextAccessor.FhirRequestContext = existingFhirRequestContext;
}
}
public static string[] GetResourceTypeNames()
{
EnsureProvider();
return(_modelInfoProvider.GetResourceTypeNames());
}
private async Task <bool> TryPopulateNewJobFields()
{
// Build query based on new search params
// Find supported, but not yet searchable params
var notYetIndexedParams = _supportedSearchParameterDefinitionManager.GetSearchParametersRequiringReindexing();
// if there are not any parameters which are supported but not yet indexed, then we have nothing to do
if (!notYetIndexedParams.Any())
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Information,
OperationOutcomeConstants.IssueType.Informational,
Resources.NoSearchParametersNeededToBeIndexed));
_reindexJobRecord.CanceledTime = Clock.UtcNow;
await MoveToFinalStatusAsync(OperationStatus.Canceled);
return(false);
}
// From the param list, get the list of necessary resources which should be
// included in our query
var resourceList = new HashSet <string>();
foreach (var param in notYetIndexedParams)
{
foreach (var baseResourceType in param.BaseResourceTypes)
{
if (baseResourceType == KnownResourceTypes.Resource)
{
resourceList.UnionWith(_modelInfoProvider.GetResourceTypeNames().ToHashSet());
// We added all possible resource types, so no need to continue
break;
}
if (baseResourceType == KnownResourceTypes.DomainResource)
{
var domainResourceChildResourceTypes = _modelInfoProvider.GetResourceTypeNames().ToHashSet();
// Remove types that inherit from Resource directly
domainResourceChildResourceTypes.Remove(KnownResourceTypes.Binary);
domainResourceChildResourceTypes.Remove(KnownResourceTypes.Bundle);
domainResourceChildResourceTypes.Remove(KnownResourceTypes.Parameters);
resourceList.UnionWith(domainResourceChildResourceTypes);
}
else
{
resourceList.UnionWith(new[] { baseResourceType });
}
}
}
foreach (var resource in resourceList)
{
_reindexJobRecord.Resources.Add(resource);
}
foreach (var searchParams in notYetIndexedParams.Select(p => p.Url.ToString()))
{
_reindexJobRecord.SearchParams.Add(searchParams);
}
await CalculateTotalAndResourceCounts();
if (_reindexJobRecord.Count == 0)
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Information,
OperationOutcomeConstants.IssueType.Informational,
Resources.NoResourcesNeedToBeReindexed));
await UpdateParametersAndCompleteJob();
return(false);
}
// Generate separate queries for each resource type and add them to query list.
foreach (string resourceType in _reindexJobRecord.Resources)
{
// Checking resource specific counts is a performance improvement,
// so if an entry for this resource failed to get added to the count dictionary, run a query anyways
if (!_reindexJobRecord.ResourceCounts.ContainsKey(resourceType) || _reindexJobRecord.ResourceCounts[resourceType] > 0)
{
var query = new ReindexJobQueryStatus(resourceType, continuationToken: null)
{
LastModified = Clock.UtcNow,
Status = OperationStatus.Queued,
};
_reindexJobRecord.QueryList.TryAdd(query, 1);
}
}
await UpdateJobAsync();
_throttleController.UpdateDatastoreUsage();
return(true);
}