public HomeController(ICalculator calc,
IScoped scoped1,
IScoped scoped2,
ITransient transient1,
ITransient transient2)
{
_Calculator = calc;
_ScopedEqual = scoped1 == scoped2;
_TransientsEqual = transient1 == transient2;
}
/// <summary>
/// Initializes a new instance of the <see cref="CosmosFhirOperationDataStore"/> class.
/// </summary>
/// <param name="containerScope">The factory for <see cref="Container"/>.</param>
/// <param name="cosmosDataStoreConfiguration">The data store configuration.</param>
/// <param name="namedCosmosCollectionConfigurationAccessor">The IOptions accessor to get a named version.</param>
/// <param name="retryExceptionPolicyFactory">The retry exception policy factory.</param>
/// <param name="queryFactory">The Query Factory</param>
/// <param name="logger">The logger.</param>
public CosmosFhirOperationDataStore(
IScoped <Container> containerScope,
CosmosDataStoreConfiguration cosmosDataStoreConfiguration,
IOptionsMonitor <CosmosCollectionConfiguration> namedCosmosCollectionConfigurationAccessor,
RetryExceptionPolicyFactory retryExceptionPolicyFactory,
ICosmosQueryFactory queryFactory,
ILogger <CosmosFhirOperationDataStore> logger)
{
EnsureArg.IsNotNull(containerScope, nameof(containerScope));
EnsureArg.IsNotNull(cosmosDataStoreConfiguration, nameof(cosmosDataStoreConfiguration));
EnsureArg.IsNotNull(namedCosmosCollectionConfigurationAccessor, nameof(namedCosmosCollectionConfigurationAccessor));
EnsureArg.IsNotNull(retryExceptionPolicyFactory, nameof(retryExceptionPolicyFactory));
EnsureArg.IsNotNull(logger, nameof(logger));
_containerScope = containerScope;
_retryExceptionPolicyFactory = retryExceptionPolicyFactory;
_queryFactory = queryFactory;
_logger = logger;
CosmosCollectionConfiguration collectionConfiguration = namedCosmosCollectionConfigurationAccessor.Get(Constants.CollectionConfigurationName);
DatabaseId = cosmosDataStoreConfiguration.DatabaseId;
CollectionId = collectionConfiguration.CollectionId;
}
/// <summary>
/// Initializes a new instance of the <see cref="CosmosFhirDataStore"/> class.
/// </summary>
/// <param name="containerScope">
/// A function that returns an <see cref="Container"/>.
/// Note that this is a function so that the lifetime of the instance is not directly controlled by the IoC container.
/// </param>
/// <param name="cosmosDataStoreConfiguration">The data store configuration.</param>
/// <param name="namedCosmosCollectionConfigurationAccessor">The IOptions accessor to get a named version.</param>
/// <param name="cosmosQueryFactory">The factory used to create the document query.</param>
/// <param name="retryExceptionPolicyFactory">The retry exception policy factory.</param>
/// <param name="logger">The logger instance.</param>
/// <param name="coreFeatures">The core feature configuration</param>
public CosmosFhirDataStore(
IScoped <Container> containerScope,
CosmosDataStoreConfiguration cosmosDataStoreConfiguration,
IOptionsMonitor <CosmosCollectionConfiguration> namedCosmosCollectionConfigurationAccessor,
ICosmosQueryFactory cosmosQueryFactory,
RetryExceptionPolicyFactory retryExceptionPolicyFactory,
ILogger <CosmosFhirDataStore> logger,
IOptions <CoreFeatureConfiguration> coreFeatures)
{
EnsureArg.IsNotNull(containerScope, nameof(containerScope));
EnsureArg.IsNotNull(cosmosDataStoreConfiguration, nameof(cosmosDataStoreConfiguration));
EnsureArg.IsNotNull(namedCosmosCollectionConfigurationAccessor, nameof(namedCosmosCollectionConfigurationAccessor));
EnsureArg.IsNotNull(cosmosQueryFactory, nameof(cosmosQueryFactory));
EnsureArg.IsNotNull(retryExceptionPolicyFactory, nameof(retryExceptionPolicyFactory));
EnsureArg.IsNotNull(logger, nameof(logger));
EnsureArg.IsNotNull(coreFeatures, nameof(coreFeatures));
_containerScope = containerScope;
_cosmosDataStoreConfiguration = cosmosDataStoreConfiguration;
_cosmosQueryFactory = cosmosQueryFactory;
_retryExceptionPolicyFactory = retryExceptionPolicyFactory;
_logger = logger;
_coreFeatures = coreFeatures.Value;
}
public ExportJobTask(
Func <IScoped <IFhirOperationDataStore> > fhirOperationDataStoreFactory,
IOptions <ExportJobConfiguration> exportJobConfiguration,
Func <IScoped <ISearchService> > searchServiceFactory,
IGroupMemberExtractor groupMemberExtractor,
IResourceToByteArraySerializer resourceToByteArraySerializer,
IExportDestinationClient exportDestinationClient,
IResourceDeserializer resourceDeserializer,
IScoped <IAnonymizerFactory> anonymizerFactory,
IMediator mediator,
IFhirRequestContextAccessor contextAccessor,
ILogger <ExportJobTask> logger)
{
EnsureArg.IsNotNull(fhirOperationDataStoreFactory, nameof(fhirOperationDataStoreFactory));
EnsureArg.IsNotNull(exportJobConfiguration?.Value, nameof(exportJobConfiguration));
EnsureArg.IsNotNull(searchServiceFactory, nameof(searchServiceFactory));
EnsureArg.IsNotNull(groupMemberExtractor, nameof(groupMemberExtractor));
EnsureArg.IsNotNull(resourceToByteArraySerializer, nameof(resourceToByteArraySerializer));
EnsureArg.IsNotNull(exportDestinationClient, nameof(exportDestinationClient));
EnsureArg.IsNotNull(resourceDeserializer, nameof(resourceDeserializer));
EnsureArg.IsNotNull(mediator, nameof(mediator));
EnsureArg.IsNotNull(contextAccessor, nameof(contextAccessor));
EnsureArg.IsNotNull(logger, nameof(logger));
_fhirOperationDataStoreFactory = fhirOperationDataStoreFactory;
_exportJobConfiguration = exportJobConfiguration.Value;
_searchServiceFactory = searchServiceFactory;
_groupMemberExtractor = groupMemberExtractor;
_resourceToByteArraySerializer = resourceToByteArraySerializer;
_resourceDeserializer = resourceDeserializer;
_exportDestinationClient = exportDestinationClient;
_anonymizerFactory = anonymizerFactory;
_mediator = mediator;
_contextAccessor = contextAccessor;
_logger = logger;
}
public RedisCache(IScoped scopedFac, ILog log)
{
this._scoped = scopedFac.GetScoped();
this._log = log;
}
public async Task <T> Run <T>(IScoped <T, IAtomicContext> command)
{
return(await Run(command.Run));
}
private async Task RunExportSearch(
ExportJobConfiguration exportJobConfiguration,
ExportJobProgress progress,
List <Tuple <string, string> > sharedQueryParametersList,
CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(exportJobConfiguration, nameof(exportJobConfiguration));
EnsureArg.IsNotNull(progress, nameof(progress));
EnsureArg.IsNotNull(sharedQueryParametersList, nameof(sharedQueryParametersList));
// Current batch will be used to organize a set of search results into a group so that they can be committed together.
string currentBatchId = progress.Page.ToString("d6");
List <Tuple <string, string> > queryParametersList = new List <Tuple <string, string> >(sharedQueryParametersList);
if (progress.ContinuationToken != null)
{
queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken));
}
if (_exportJobRecord.ExportType == ExportJobType.Patient)
{
queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.Type, KnownResourceTypes.Patient));
}
else if (_exportJobRecord.ExportType == ExportJobType.All && !string.IsNullOrEmpty(_exportJobRecord.ResourceType))
{
queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.Type, _exportJobRecord.ResourceType));
}
// Process the export if:
// 1. There is continuation token, which means there is more resource to be exported.
// 2. There is no continuation token but the page is 0, which means it's the initial export.
while (progress.ContinuationToken != null || progress.Page == 0)
{
SearchResult searchResult = null;
// Search and process the results.
switch (_exportJobRecord.ExportType)
{
case ExportJobType.All:
case ExportJobType.Patient:
using (IScoped <ISearchService> searchService = _searchServiceFactory())
{
searchResult = await searchService.Value.SearchAsync(
resourceType : null,
queryParametersList,
cancellationToken);
}
break;
case ExportJobType.Group:
searchResult = await GetGroupPatients(
_exportJobRecord.GroupId,
queryParametersList,
_exportJobRecord.QueuedTime,
cancellationToken);
break;
}
if (_exportJobRecord.ExportType == ExportJobType.Patient || _exportJobRecord.ExportType == ExportJobType.Group)
{
uint resultIndex = 0;
foreach (SearchResultEntry result in searchResult.Results)
{
// If a job is resumed in the middle of processing patient compartment resources it will skip patients it has already exported compartment information for.
// This assumes the order of the search results is the same every time the same search is performed.
if (progress.SubSearch != null && result.Resource.ResourceId != progress.SubSearch.TriggeringResourceId)
{
resultIndex++;
continue;
}
if (progress.SubSearch == null)
{
progress.NewSubSearch(result.Resource.ResourceId);
}
await RunExportCompartmentSearch(exportJobConfiguration, progress.SubSearch, sharedQueryParametersList, cancellationToken, currentBatchId + ":" + resultIndex.ToString("d6"));
resultIndex++;
progress.ClearSubSearch();
}
}
await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, cancellationToken);
if (searchResult.ContinuationToken == null)
{
// No more continuation token, we are done.
break;
}
await ProcessProgressChange(
exportJobConfiguration,
progress,
queryParametersList,
searchResult.ContinuationToken,
forceCommit : _exportJobRecord.ExportType == ExportJobType.Patient || _exportJobRecord.ExportType == ExportJobType.Group,
cancellationToken);
currentBatchId = progress.Page.ToString("d6");
}
// Commit one last time for any pending changes.
await _exportDestinationClient.CommitAsync(exportJobConfiguration, cancellationToken);
}
public ReindexSearchTests(FhirStorageTestsFixture fixture)
{
_scopedDataStore = fixture.DataStore.CreateMockScope();
_searchService = fixture.SearchService.CreateMockScope();
_searchParameterDefinitionManager = fixture.SearchParameterDefinitionManager;
}
public async Task <T> Run <T>(IScoped <T, ITransactionContext> transaction)
{
return(await Run(transaction.Run));
}
/// <inheritdoc />
public async Task ExecuteAsync(ReindexJobRecord reindexJobRecord, WeakETag weakETag, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(reindexJobRecord, nameof(reindexJobRecord));
EnsureArg.IsNotNull(weakETag, nameof(weakETag));
_reindexJobRecord = reindexJobRecord;
_weakETag = weakETag;
var jobSemaphore = new SemaphoreSlim(1, 1);
var existingFhirRequestContext = _contextAccessor.FhirRequestContext;
try
{
// Add a request context so Datastore consumption can be added
var fhirRequestContext = new FhirRequestContext(
method: OperationsConstants.Reindex,
uriString: "$reindex",
baseUriString: "$reindex",
correlationId: _reindexJobRecord.Id,
requestHeaders: new Dictionary <string, StringValues>(),
responseHeaders: new Dictionary <string, StringValues>())
{
IsBackgroundTask = true,
AuditEventType = OperationsConstants.Reindex,
};
_contextAccessor.FhirRequestContext = fhirRequestContext;
using (IScoped <IFhirDataStore> store = _fhirDataStoreFactory())
{
var provisionedCapacity = await store.Value.GetProvisionedDataStoreCapacityAsync(cancellationToken);
_throttleController.Initialize(_reindexJobRecord, provisionedCapacity);
}
if (_reindexJobRecord.Status != OperationStatus.Running ||
_reindexJobRecord.StartTime == null)
{
// update job record to running
_reindexJobRecord.Status = OperationStatus.Running;
_reindexJobRecord.StartTime = Clock.UtcNow;
await UpdateJobAsync(cancellationToken);
}
// If we are resuming a job, we can detect that by checking the progress info from the job record.
// If no queries have been added to the progress then this is a new job
if (_reindexJobRecord.QueryList?.Count == 0)
{
// Build query based on new search params
// Find supported, but not yet searchable params
var notYetIndexedParams = _supportedSearchParameterDefinitionManager.GetSearchParametersRequiringReindexing();
// if there are not any parameters which are supported but not yet indexed, then we have nothing to do
if (!notYetIndexedParams.Any())
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Information,
OperationOutcomeConstants.IssueType.Informational,
Resources.NoSearchParametersNeededToBeIndexed));
_reindexJobRecord.CanceledTime = DateTimeOffset.UtcNow;
await CompleteJobAsync(OperationStatus.Canceled, cancellationToken);
return;
}
// From the param list, get the list of necessary resources which should be
// included in our query
var resourceList = new HashSet <string>();
foreach (var param in notYetIndexedParams)
{
foreach (var baseResourceType in param.BaseResourceTypes)
{
if (baseResourceType == KnownResourceTypes.Resource)
{
resourceList.UnionWith(_modelInfoProvider.GetResourceTypeNames().ToHashSet());
// We added all possible resource types, so no need to continue
break;
}
if (baseResourceType == KnownResourceTypes.DomainResource)
{
var domainResourceChildResourceTypes = _modelInfoProvider.GetResourceTypeNames().ToHashSet();
// Remove types that inherit from Resource directly
domainResourceChildResourceTypes.Remove(KnownResourceTypes.Binary);
domainResourceChildResourceTypes.Remove(KnownResourceTypes.Bundle);
domainResourceChildResourceTypes.Remove(KnownResourceTypes.Parameters);
resourceList.UnionWith(domainResourceChildResourceTypes);
}
else
{
resourceList.UnionWith(new[] { baseResourceType });
}
}
}
_reindexJobRecord.Resources.AddRange(resourceList);
_reindexJobRecord.SearchParams.AddRange(notYetIndexedParams.Select(p => p.Url.ToString()));
await CalculateTotalAndResourceCounts(cancellationToken);
if (_reindexJobRecord.Count == 0)
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Information,
OperationOutcomeConstants.IssueType.Informational,
Resources.NoResourcesNeedToBeReindexed));
await UpdateParametersAndCompleteJob(cancellationToken);
return;
}
// Generate separate queries for each resource type and add them to query list.
foreach (string resourceType in _reindexJobRecord.Resources)
{
// Checking resource specific counts is a performance improvement,
// so if an entry for this resource failed to get added to the count dictionary, run a query anyways
if (!_reindexJobRecord.ResourceCounts.ContainsKey(resourceType) || _reindexJobRecord.ResourceCounts[resourceType] > 0)
{
var query = new ReindexJobQueryStatus(resourceType, continuationToken: null)
{
LastModified = Clock.UtcNow,
Status = OperationStatus.Queued,
};
_reindexJobRecord.QueryList.TryAdd(query, 1);
}
}
await UpdateJobAsync(cancellationToken);
_throttleController.UpdateDatastoreUsage();
}
var queryTasks = new List <Task <ReindexJobQueryStatus> >();
var queryCancellationTokens = new Dictionary <ReindexJobQueryStatus, CancellationTokenSource>();
// while not all queries are finished
while (_reindexJobRecord.QueryList.Keys.Where(q =>
q.Status == OperationStatus.Queued ||
q.Status == OperationStatus.Running).Any())
{
if (_reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Queued).Any())
{
// grab the next query from the list which is labeled as queued and run it
var query = _reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Queued).OrderBy(q => q.LastModified).FirstOrDefault();
CancellationTokenSource queryTokensSource = new CancellationTokenSource();
queryCancellationTokens.TryAdd(query, queryTokensSource);
#pragma warning disable CS4014 // Suppressed as we want to continue execution and begin processing the next query while this continues to run
queryTasks.Add(ProcessQueryAsync(query, jobSemaphore, queryTokensSource.Token));
#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
_logger.LogInformation($"Reindex job task created {queryTasks.Count} Tasks");
}
// reset stale queries to pending
var staleQueries = _reindexJobRecord.QueryList.Keys.Where(
q => q.Status == OperationStatus.Running && q.LastModified < Clock.UtcNow - _reindexJobConfiguration.JobHeartbeatTimeoutThreshold);
foreach (var staleQuery in staleQueries)
{
await jobSemaphore.WaitAsync();
try
{
// if this query has a created task, cancel it
if (queryCancellationTokens.TryGetValue(staleQuery, out var tokenSource))
{
try
{
tokenSource.Cancel(false);
}
catch
{
// may throw exception if the task is disposed
}
}
staleQuery.Status = OperationStatus.Queued;
await UpdateJobAsync(cancellationToken);
}
finally
{
jobSemaphore.Release();
}
}
var averageDbConsumption = _throttleController.UpdateDatastoreUsage();
_logger.LogInformation($"Reindex avaerage DB consumption: {averageDbConsumption}");
var throttleDelayTime = _throttleController.GetThrottleBasedDelay();
_logger.LogInformation($"Reindex throttle delay: {throttleDelayTime}");
await Task.Delay(_reindexJobRecord.QueryDelayIntervalInMilliseconds + throttleDelayTime);
// Remove all finished tasks from the collections of tasks
// and cancellationTokens
if (queryTasks.Count >= reindexJobRecord.MaximumConcurrency)
{
var taskArray = queryTasks.ToArray();
Task.WaitAny(taskArray);
var finishedTasks = queryTasks.Where(t => t.IsCompleted).ToArray();
foreach (var finishedTask in finishedTasks)
{
queryTasks.Remove(finishedTask);
queryCancellationTokens.Remove(await finishedTask);
}
}
// if our received CancellationToken is cancelled we should
// pass that cancellation request onto all the cancellationTokens
// for the currently executing threads
if (cancellationToken.IsCancellationRequested)
{
foreach (var tokenSource in queryCancellationTokens.Values)
{
tokenSource.Cancel(false);
}
}
}
Task.WaitAll(queryTasks.ToArray());
await jobSemaphore.WaitAsync();
try
{
await CheckJobCompletionStatus(cancellationToken);
}
finally
{
jobSemaphore.Release();
}
}
catch (JobConflictException)
{
// The reindex job was updated externally.
_logger.LogInformation("The job was updated by another process.");
}
catch (Exception ex)
{
await jobSemaphore.WaitAsync();
try
{
_reindexJobRecord.Error.Add(new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Error,
OperationOutcomeConstants.IssueType.Exception,
ex.Message));
_reindexJobRecord.FailureCount++;
_logger.LogError(ex, "Encountered an unhandled exception. The job failure count increased to {failureCount}.", _reindexJobRecord.FailureCount);
await UpdateJobAsync(cancellationToken);
if (_reindexJobRecord.FailureCount >= _reindexJobConfiguration.ConsecutiveFailuresThreshold)
{
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
else
{
_reindexJobRecord.Status = OperationStatus.Queued;
await UpdateJobAsync(cancellationToken);
}
}
finally
{
jobSemaphore.Release();
}
}
finally
{
jobSemaphore.Dispose();
_contextAccessor.FhirRequestContext = existingFhirRequestContext;
}
}
/// <inheritdoc />
public async Task ExecuteAsync(ReindexJobRecord reindexJobRecord, WeakETag weakETag, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(reindexJobRecord, nameof(reindexJobRecord));
EnsureArg.IsNotNull(weakETag, nameof(weakETag));
if (_reindexJobRecord != null)
{
throw new NotSupportedException($"{nameof(ReindexJobTask)} can work only on one {nameof(reindexJobRecord)}. Please create new {nameof(ReindexJobTask)} to process this instance of {nameof(reindexJobRecord)}");
}
_reindexJobRecord = reindexJobRecord;
_weakETag = weakETag;
_jobSemaphore = new SemaphoreSlim(1, 1);
_cancellationToken = cancellationToken;
var originalRequestContext = _contextAccessor.RequestContext;
try
{
// Add a request context so Datastore consumption can be added
var fhirRequestContext = new FhirRequestContext(
method: OperationsConstants.Reindex,
uriString: "$reindex",
baseUriString: "$reindex",
correlationId: _reindexJobRecord.Id,
requestHeaders: new Dictionary <string, StringValues>(),
responseHeaders: new Dictionary <string, StringValues>())
{
IsBackgroundTask = true,
AuditEventType = OperationsConstants.Reindex,
};
_contextAccessor.RequestContext = fhirRequestContext;
if (reindexJobRecord.TargetDataStoreUsagePercentage != null &&
reindexJobRecord.TargetDataStoreUsagePercentage > 0)
{
using (IScoped <IFhirDataStore> store = _fhirDataStoreFactory.Invoke())
{
var provisionedCapacity = await store.Value.GetProvisionedDataStoreCapacityAsync(_cancellationToken);
_throttleController.Initialize(_reindexJobRecord, provisionedCapacity);
}
}
else
{
_throttleController.Initialize(_reindexJobRecord, null);
}
// If we are resuming a job, we can detect that by checking the progress info from the job record.
// If no queries have been added to the progress then this is a new job
if (_reindexJobRecord.QueryList?.Count == 0)
{
if (!await TryPopulateNewJobFields())
{
return;
}
}
if (_reindexJobRecord.Status != OperationStatus.Running || _reindexJobRecord.StartTime == null)
{
// update job record to running
_reindexJobRecord.Status = OperationStatus.Running;
_reindexJobRecord.StartTime = Clock.UtcNow;
await UpdateJobAsync();
}
await ProcessJob();
await _jobSemaphore.WaitAsync(_cancellationToken);
try
{
await CheckJobCompletionStatus();
}
finally
{
_jobSemaphore.Release();
}
}
catch (JobConflictException)
{
// The reindex job was updated externally.
_logger.LogInformation("The job was updated by another process.");
}
catch (OperationCanceledException)
{
_logger.LogInformation("The reindex job was canceled.");
}
catch (Exception ex)
{
await HandleException(ex);
}
finally
{
_jobSemaphore.Dispose();
_contextAccessor.RequestContext = originalRequestContext;
}
}
public IncreaseService(ISingleton singleton, IScoped scoped, ITransient transient)
{
this.singleton = singleton;
this.scoped = scoped;
this.transient = transient;
}
public ProductController(ITransient transient1, ITransient transient2, ISingleton singleton1, ISingleton singleton2, IScoped scoped1, IScoped scoped2)
{
_transient1 = transient1;
_transient2 = transient2;
_singleton1 = singleton1;
_singleton2 = singleton2;
_scoped1 = scoped1;
_scoped2 = scoped2;
}
public HomeController(ITransient transientService1, ITransient transientService2, ISingleton singletonService1, ISingleton singletonService2, IScoped scopedService1, IScoped scopedService2)
{
this.transientService1 = transientService1;
this.transientService2 = transientService2;
this.singletonService1 = singletonService1;
this.singletonService2 = singletonService2;
this.scopedService1 = scopedService1;
this.scopedService2 = scopedService2;
}
public DIService(IScoped scoped, ITransient transient, ISingleton singleton)
{
this.scoped = scoped;
this.transient = transient;
this.singleton = singleton;
}
public async Task Run(IScoped <IAtomicContext> scoped)
{
await Run(scoped.Run);
}
public async Task <IReadOnlyCollection <ResourceSearchParameterStatus> > GetSearchParameterStatuses(CancellationToken cancellationToken)
{
// If the search parameter table in SQL does not yet contain status columns
if (_schemaInformation.Current < SchemaVersionConstants.SearchParameterStatusSchemaVersion)
{
// Get status information from file.
return(await _filebasedSearchParameterStatusDataStore.GetSearchParameterStatuses(cancellationToken));
}
using (IScoped <SqlConnectionWrapperFactory> scopedSqlConnectionWrapperFactory = _scopedSqlConnectionWrapperFactory())
using (SqlConnectionWrapper sqlConnectionWrapper = await scopedSqlConnectionWrapperFactory.Value.ObtainSqlConnectionWrapperAsync(cancellationToken, true))
using (SqlCommandWrapper sqlCommandWrapper = sqlConnectionWrapper.CreateSqlCommand())
{
VLatest.GetSearchParamStatuses.PopulateCommand(sqlCommandWrapper);
var parameterStatuses = new List <ResourceSearchParameterStatus>();
using (SqlDataReader sqlDataReader = await sqlCommandWrapper.ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken))
{
while (await sqlDataReader.ReadAsync(cancellationToken))
{
short id;
string uri;
string stringStatus;
DateTimeOffset?lastUpdated;
bool? isPartiallySupported;
ResourceSearchParameterStatus resourceSearchParameterStatus;
if (_schemaInformation.Current >= SchemaVersionConstants.SearchParameterSynchronizationVersion)
{
(id, uri, stringStatus, lastUpdated, isPartiallySupported) = sqlDataReader.ReadRow(
VLatest.SearchParam.SearchParamId,
VLatest.SearchParam.Uri,
VLatest.SearchParam.Status,
VLatest.SearchParam.LastUpdated,
VLatest.SearchParam.IsPartiallySupported);
if (string.IsNullOrEmpty(stringStatus) || lastUpdated == null || isPartiallySupported == null)
{
// These columns are nullable because they are added to dbo.SearchParam in a later schema version.
// They should be populated as soon as they are added to the table and should never be null.
throw new SearchParameterNotSupportedException(Resources.SearchParameterStatusShouldNotBeNull);
}
var status = Enum.Parse <SearchParameterStatus>(stringStatus, true);
resourceSearchParameterStatus = new SqlServerResourceSearchParameterStatus
{
Id = id,
Uri = new Uri(uri),
Status = status,
IsPartiallySupported = (bool)isPartiallySupported,
LastUpdated = (DateTimeOffset)lastUpdated,
};
}
else
{
(uri, stringStatus, lastUpdated, isPartiallySupported) = sqlDataReader.ReadRow(
VLatest.SearchParam.Uri,
VLatest.SearchParam.Status,
VLatest.SearchParam.LastUpdated,
VLatest.SearchParam.IsPartiallySupported);
if (string.IsNullOrEmpty(stringStatus) || lastUpdated == null || isPartiallySupported == null)
{
// These columns are nullable because they are added to dbo.SearchParam in a later schema version.
// They should be populated as soon as they are added to the table and should never be null.
throw new SearchParameterNotSupportedException(Resources.SearchParameterStatusShouldNotBeNull);
}
var status = Enum.Parse <SearchParameterStatus>(stringStatus, true);
resourceSearchParameterStatus = new ResourceSearchParameterStatus
{
Uri = new Uri(uri),
Status = status,
IsPartiallySupported = (bool)isPartiallySupported,
LastUpdated = (DateTimeOffset)lastUpdated,
};
}
if (_schemaInformation.Current >= SchemaVersionConstants.AddMinMaxForDateAndStringSearchParamVersion)
{
// For schema versions starting from AddMinMaxForDateAndStringSearchParamVersion we will check
// whether the corresponding type of the search parameter is supported.
SearchParameterInfo paramInfo = null;
try
{
paramInfo = _searchParameterDefinitionManager.GetSearchParameter(resourceSearchParameterStatus.Uri.OriginalString);
}
catch (SearchParameterNotSupportedException)
{
}
if (paramInfo != null && SqlServerSortingValidator.SupportedSortParamTypes.Contains(paramInfo.Type))
{
resourceSearchParameterStatus.SortStatus = SortParameterStatus.Enabled;
}
else
{
resourceSearchParameterStatus.SortStatus = SortParameterStatus.Disabled;
}
}
else
{
if (_sortingValidator.SupportedParameterUris.Contains(resourceSearchParameterStatus.Uri))
{
resourceSearchParameterStatus.SortStatus = SortParameterStatus.Enabled;
}
else
{
resourceSearchParameterStatus.SortStatus = SortParameterStatus.Disabled;
}
}
parameterStatuses.Add(resourceSearchParameterStatus);
}
}
return(parameterStatuses);
}
}
private async Task ProcessJob()
{
var queryTasks = new List <Task <ReindexJobQueryStatus> >();
var queryCancellationTokens = new Dictionary <ReindexJobQueryStatus, CancellationTokenSource>();
// while not all queries are finished
while (_reindexJobRecord.QueryList.Keys.Where(q =>
q.Status == OperationStatus.Queued ||
q.Status == OperationStatus.Running).Any())
{
if (_reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Queued).Any())
{
// grab the next query from the list which is labeled as queued and run it
var query = _reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Queued).OrderBy(q => q.LastModified).FirstOrDefault();
CancellationTokenSource queryTokensSource = new CancellationTokenSource();
queryCancellationTokens.TryAdd(query, queryTokensSource);
// We don't await ProcessQuery, so query status can or can not be changed inside immediately
// In some cases we can go th6rough whole loop and pick same query from query list.
// To prevent that we marking query as running here and not inside ProcessQuery code.
query.Status = OperationStatus.Running;
query.LastModified = Clock.UtcNow;
#pragma warning disable CS4014 // Suppressed as we want to continue execution and begin processing the next query while this continues to run
queryTasks.Add(ProcessQueryAsync(query, queryTokensSource.Token));
#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
_logger.LogInformation($"Reindex job task created {queryTasks.Count} Tasks");
}
// reset stale queries to pending
var staleQueries = _reindexJobRecord.QueryList.Keys.Where(
q => q.Status == OperationStatus.Running && q.LastModified < Clock.UtcNow - _reindexJobConfiguration.JobHeartbeatTimeoutThreshold);
foreach (var staleQuery in staleQueries)
{
await _jobSemaphore.WaitAsync(_cancellationToken);
try
{
// if this query has a created task, cancel it
if (queryCancellationTokens.TryGetValue(staleQuery, out var tokenSource))
{
try
{
tokenSource.Cancel(false);
}
catch
{
// may throw exception if the task is disposed
}
}
staleQuery.Status = OperationStatus.Queued;
await UpdateJobAsync();
}
finally
{
_jobSemaphore.Release();
}
}
var averageDbConsumption = _throttleController.UpdateDatastoreUsage();
_logger.LogInformation($"Reindex avaerage DB consumption: {averageDbConsumption}");
var throttleDelayTime = _throttleController.GetThrottleBasedDelay();
_logger.LogInformation($"Reindex throttle delay: {throttleDelayTime}");
await Task.Delay(_reindexJobRecord.QueryDelayIntervalInMilliseconds + throttleDelayTime, _cancellationToken);
// Remove all finished tasks from the collections of tasks
// and cancellationTokens
if (queryTasks.Count >= _reindexJobRecord.MaximumConcurrency)
{
var taskArray = queryTasks.ToArray();
Task.WaitAny(taskArray, _cancellationToken);
var finishedTasks = queryTasks.Where(t => t.IsCompleted).ToArray();
foreach (var finishedTask in finishedTasks)
{
queryTasks.Remove(finishedTask);
queryCancellationTokens.Remove(await finishedTask);
}
}
// for most cases if another process updates the job (such as a DELETE request)
// the _etag change will cause a JobConflict exception and this task will be aborted
// but here we add one more check before attempting to mark the job as complete,
// or starting another iteration of the loop
await _jobSemaphore.WaitAsync();
try
{
using (IScoped <IFhirOperationDataStore> store = _fhirOperationDataStoreFactory.Invoke())
{
var wrapper = await store.Value.GetReindexJobByIdAsync(_reindexJobRecord.Id, _cancellationToken);
_weakETag = wrapper.ETag;
_reindexJobRecord.Status = wrapper.JobRecord.Status;
}
}
catch (Exception)
{
// if something went wrong with fetching job status, we shouldn't fail process loop.
}
finally
{
_jobSemaphore.Release();
}
// if our received CancellationToken is cancelled, or the job has been marked canceled we should
// pass that cancellation request onto all the cancellationTokens
// for the currently executing threads
if (_cancellationToken.IsCancellationRequested || _reindexJobRecord.Status == OperationStatus.Canceled)
{
foreach (var tokenSource in queryCancellationTokens.Values)
{
tokenSource.Cancel(false);
}
_logger.LogInformation("Reindex Job canceled.");
throw new OperationCanceledException("ReindexJob canceled.");
}
}
Task.WaitAll(queryTasks.ToArray(), _cancellationToken);
}
public GenerateController(IScoped scoped, ITransient transient, ISingelton singelton)
{
_scoped = scoped;
_transient = transient;
_singelton = singelton;
}
public ScopesController(IServiceOne testingServiceOne, IServiceTwo testingServiceTwo, ISingleton singleton, IScoped scoped, ITransient transient)
{
this.testingServiceOne = testingServiceOne;
this.testingServiceTwo = testingServiceTwo;
this.singleton = singleton;
this.scoped = scoped;
this.transient = transient;
}
/// <inheritdoc />
public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETag, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(exportJobRecord, nameof(exportJobRecord));
_exportJobRecord = exportJobRecord;
_weakETag = weakETag;
try
{
ExportJobConfiguration exportJobConfiguration = _exportJobConfiguration;
string connectionHash = string.IsNullOrEmpty(_exportJobConfiguration.StorageAccountConnection) ?
string.Empty :
Microsoft.Health.Core.Extensions.StringExtensions.ComputeHash(_exportJobConfiguration.StorageAccountConnection);
if (string.IsNullOrEmpty(exportJobRecord.StorageAccountUri))
{
if (!string.Equals(exportJobRecord.StorageAccountConnectionHash, connectionHash, StringComparison.Ordinal))
{
throw new DestinationConnectionException("Storage account connection string was updated during an export job.", HttpStatusCode.BadRequest);
}
}
else
{
exportJobConfiguration = new ExportJobConfiguration();
exportJobConfiguration.Enabled = _exportJobConfiguration.Enabled;
exportJobConfiguration.StorageAccountUri = exportJobRecord.StorageAccountUri;
}
// Connect to export destination using appropriate client.
await _exportDestinationClient.ConnectAsync(exportJobConfiguration, cancellationToken, _exportJobRecord.Id);
// If we are resuming a job, we can detect that by checking the progress info from the job record.
// If it is null, then we know we are processing a new job.
if (_exportJobRecord.Progress == null)
{
_exportJobRecord.Progress = new ExportJobProgress(continuationToken: null, page: 0);
}
ExportJobProgress progress = _exportJobRecord.Progress;
// Current batch will be used to organize a set of search results into a group so that they can be committed together.
uint currentBatchId = progress.Page;
// The intial list of query parameters will not have a continutation token. We will add that later if we get one back
// from the search result.
var queryParametersList = new List <Tuple <string, string> >()
{
Tuple.Create(KnownQueryParameterNames.Count, _exportJobConfiguration.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)),
Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{_exportJobRecord.QueuedTime.ToString("o", CultureInfo.InvariantCulture)}"),
};
if (_exportJobRecord.Since != null)
{
queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.LastUpdated, $"ge{_exportJobRecord.Since}"));
}
// Process the export if:
// 1. There is continuation token, which means there is more resource to be exported.
// 2. There is no continuation token but the page is 0, which means it's the initial export.
while (progress.ContinuationToken != null || progress.Page == 0)
{
SearchResult searchResult;
// Search and process the results.
using (IScoped <ISearchService> searchService = _searchServiceFactory())
{
searchResult = await searchService.Value.SearchAsync(
_exportJobRecord.ResourceType,
queryParametersList,
cancellationToken);
}
await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, cancellationToken);
if (searchResult.ContinuationToken == null)
{
// No more continuation token, we are done.
break;
}
// Update the continuation token in local cache and queryParams.
// We will add or udpate the continuation token to the end of the query parameters list.
progress.UpdateContinuationToken(searchResult.ContinuationToken);
if (queryParametersList[queryParametersList.Count - 1].Item1 == KnownQueryParameterNames.ContinuationToken)
{
queryParametersList[queryParametersList.Count - 1] = Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken);
}
else
{
queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken));
}
if (progress.Page % _exportJobConfiguration.NumberOfPagesPerCommit == 0)
{
// Commit the changes.
await _exportDestinationClient.CommitAsync(exportJobConfiguration, cancellationToken);
// Update the job record.
await UpdateJobRecordAsync(cancellationToken);
currentBatchId = progress.Page;
}
}
// Commit one last time for any pending changes.
await _exportDestinationClient.CommitAsync(exportJobConfiguration, cancellationToken);
await CompleteJobAsync(OperationStatus.Completed, cancellationToken);
_logger.LogTrace("Successfully completed the job.");
}
catch (JobConflictException)
{
// The export job was updated externally. There might be some additional resources that were exported
// but we will not be updating the job record.
_logger.LogTrace("The job was updated by another process.");
}
catch (DestinationConnectionException dce)
{
_logger.LogError(dce, "Can't connect to destination. The job will be marked as failed.");
_exportJobRecord.FailureDetails = new JobFailureDetails(dce.Message, dce.StatusCode);
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
catch (Exception ex)
{
// The job has encountered an error it cannot recover from.
// Try to update the job to failed state.
_logger.LogError(ex, "Encountered an unhandled exception. The job will be marked as failed.");
_exportJobRecord.FailureDetails = new JobFailureDetails(Resources.UnknownError, HttpStatusCode.InternalServerError);
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
}
/// <inheritdoc />
public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETag, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(exportJobRecord, nameof(exportJobRecord));
_exportJobRecord = exportJobRecord;
_weakETag = weakETag;
try
{
// Get destination type from secret store and connect to the destination using appropriate client.
await GetDestinationInfoAndConnectAsync(cancellationToken);
// If we are resuming a job, we can detect that by checking the progress info from the job record.
// If it is null, then we know we are processing a new job.
if (_exportJobRecord.Progress == null)
{
_exportJobRecord.Progress = new ExportJobProgress(continuationToken: null, page: 0);
}
ExportJobProgress progress = _exportJobRecord.Progress;
// Current batch will be used to organize a set of search results into a group so that they can be committed together.
uint currentBatchId = progress.Page;
// The first item is placeholder for continuation token so that it can be updated efficiently later.
var queryParameters = new Tuple <string, string>[]
{
Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken),
Tuple.Create(KnownQueryParameterNames.Count, _exportJobConfiguration.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)),
Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{_exportJobRecord.QueuedTime.ToString("o", CultureInfo.InvariantCulture)}"),
};
// Process the export if:
// 1. There is continuation token, which means there is more resource to be exported.
// 2. There is no continuation token but the page is 0, which means it's the initial export.
while (progress.ContinuationToken != null || progress.Page == 0)
{
SearchResult searchResult;
// Search and process the results.
using (IScoped <ISearchService> searchService = _searchServiceFactory())
{
// If the continuation token is null, then we will exclude it. Calculate the offset and count to be passed in.
int offset = queryParameters[0].Item2 == null ? 1 : 0;
searchResult = await searchService.Value.SearchAsync(
_exportJobRecord.ResourceType,
new ArraySegment <Tuple <string, string> >(queryParameters, offset, queryParameters.Length - offset),
cancellationToken);
}
await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, cancellationToken);
if (searchResult.ContinuationToken == null)
{
// No more continuation token, we are done.
break;
}
// Update the continuation token (local cache).
progress.UpdateContinuationToken(searchResult.ContinuationToken);
queryParameters[0] = Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken);
if (progress.Page % _exportJobConfiguration.NumberOfPagesPerCommit == 0)
{
// Commit the changes.
await _exportDestinationClient.CommitAsync(cancellationToken);
// Update the job record.
await UpdateJobRecordAsync(cancellationToken);
currentBatchId = progress.Page;
}
}
// Commit one last time for any pending changes.
await _exportDestinationClient.CommitAsync(cancellationToken);
await CompleteJobAsync(OperationStatus.Completed, cancellationToken);
_logger.LogTrace("Successfully completed the job.");
try
{
// Best effort to delete the secret. If it fails to delete, then move on.
await _secretStore.DeleteSecretAsync(_exportJobRecord.SecretName, cancellationToken);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to delete the secret.");
}
}
catch (JobConflictException)
{
// The export job was updated externally. There might be some additional resources that were exported
// but we will not be updating the job record.
_logger.LogTrace("The job was updated by another process.");
}
catch (Exception ex)
{
// The job has encountered an error it cannot recover from.
// Try to update the job to failed state.
_logger.LogError(ex, "Encountered an unhandled exception. The job will be marked as failed.");
await CompleteJobAsync(OperationStatus.Failed, cancellationToken);
}
}
public async Task Run(IScoped <ITransactionContext> scoped)
{
await Run(scoped.Run);
}
private async Task SearchWithFilter(
ExportJobConfiguration exportJobConfiguration,
ExportJobProgress progress,
string resourceType,
List <Tuple <string, string> > queryParametersList,
List <Tuple <string, string> > sharedQueryParametersList,
IAnonymizer anonymizer,
string batchIdPrefix,
CancellationToken cancellationToken)
{
// Current batch will be used to organize a set of search results into a group so that they can be committed together.
string currentBatchId = batchIdPrefix + progress.Page.ToString("d6");
// Process the export if:
// 1. There is continuation token, which means there is more resource to be exported.
// 2. There is no continuation token but the page is 0, which means it's the initial export.
while (progress.ContinuationToken != null || progress.Page == 0)
{
SearchResult searchResult = null;
// Search and process the results.
switch (_exportJobRecord.ExportType)
{
case ExportJobType.All:
case ExportJobType.Patient:
using (IScoped <ISearchService> searchService = _searchServiceFactory())
{
searchResult = await searchService.Value.SearchAsync(
resourceType : resourceType,
queryParametersList,
cancellationToken);
}
break;
case ExportJobType.Group:
searchResult = await GetGroupPatients(
_exportJobRecord.GroupId,
queryParametersList,
_exportJobRecord.QueuedTime,
cancellationToken);
break;
}
if (_exportJobRecord.ExportType == ExportJobType.Patient || _exportJobRecord.ExportType == ExportJobType.Group)
{
uint resultIndex = 0;
foreach (SearchResultEntry result in searchResult.Results)
{
// If a job is resumed in the middle of processing patient compartment resources it will skip patients it has already exported compartment information for.
// This assumes the order of the search results is the same every time the same search is performed.
if (progress.SubSearch != null && result.Resource.ResourceId != progress.SubSearch.TriggeringResourceId)
{
resultIndex++;
continue;
}
if (progress.SubSearch == null)
{
progress.NewSubSearch(result.Resource.ResourceId);
}
await RunExportCompartmentSearch(exportJobConfiguration, progress.SubSearch, sharedQueryParametersList, anonymizer, cancellationToken, currentBatchId + ":" + resultIndex.ToString("d6"));
resultIndex++;
progress.ClearSubSearch();
}
}
// Skips processing top level search results if the job only requested resources from the compartments of patients, but didn't want the patients.
if (_exportJobRecord.ExportType == ExportJobType.All ||
string.IsNullOrWhiteSpace(_exportJobRecord.ResourceType) ||
_exportJobRecord.ResourceType.Contains(KnownResourceTypes.Patient, StringComparison.OrdinalIgnoreCase))
{
await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, anonymizer, cancellationToken);
}
if (searchResult.ContinuationToken == null)
{
// No more continuation token, we are done.
break;
}
await ProcessProgressChange(
exportJobConfiguration,
progress,
queryParametersList,
searchResult.ContinuationToken,
forceCommit : _exportJobRecord.ExportType == ExportJobType.Patient || _exportJobRecord.ExportType == ExportJobType.Group,
cancellationToken);
currentBatchId = batchIdPrefix + progress.Page.ToString("d6");
}
// Commit one last time for any pending changes.
await _exportDestinationClient.CommitAsync(exportJobConfiguration, cancellationToken);
}
public async Task ValidateSearchParamterInput(SearchParameter searchParam, string method, CancellationToken cancellationToken)
{
if (await _authorizationService.CheckAccess(DataActions.Reindex) != DataActions.Reindex)
{
throw new UnauthorizedFhirActionException();
}
// check if reindex job is running
using (IScoped <IFhirOperationDataStore> fhirOperationDataStore = _fhirOperationDataStoreFactory())
{
(var activeReindexJobs, var reindexJobId) = await fhirOperationDataStore.Value.CheckActiveReindexJobsAsync(cancellationToken);
if (activeReindexJobs)
{
throw new JobConflictException(string.Format(Resources.ChangesToSearchParametersNotAllowedWhileReindexing, reindexJobId));
}
}
var validationFailures = new List <ValidationFailure>();
if (string.IsNullOrEmpty(searchParam.Url))
{
validationFailures.Add(
new ValidationFailure(nameof(Base.TypeName), Resources.SearchParameterDefinitionInvalidMissingUri));
}
else
{
try
{
_searchParameterDefinitionManager.GetSearchParameter(new Uri(searchParam.Url));
// If a post, then it is a creation of a new search parameter
// only allow this if no other parameters exist with the same Uri
if (method.Equals(HttpPostName, StringComparison.OrdinalIgnoreCase))
{
// if no exception is thrown, then the search parameter with the same Uri was found
// and this is a conflict
validationFailures.Add(
new ValidationFailure(
nameof(searchParam.Url),
string.Format(Resources.SearchParameterDefinitionDuplicatedEntry, searchParam.Url)));
}
}
catch (FormatException)
{
validationFailures.Add(
new ValidationFailure(
nameof(searchParam.Url),
string.Format(Resources.SearchParameterDefinitionInvalidDefinitionUri, searchParam.Url)));
}
catch (SearchParameterNotSupportedException)
{
// if thrown, then the search parameter is not found
// if a PUT, then we should be updating an exsting paramter, but it was not found
if (method.Equals(HttpPutName, StringComparison.OrdinalIgnoreCase) ||
method.Equals(HttpDeleteName, StringComparison.OrdinalIgnoreCase))
{
// if an exception above was thrown, then the search parameter with the same Uri was not found
// and DELETE or PUT can only run on existing parameter
validationFailures.Add(
new ValidationFailure(
nameof(searchParam.Url),
string.Format(Resources.SearchParameterDefinitionNotFound, searchParam.Url)));
}
}
}
// validate that the url does not correspond to a search param from the spec
// TODO: still need a method to determine spec defined search params
// validation of the fhir path
// TODO: separate user story for this validation
if (validationFailures.Any())
{
throw new ResourceNotValidException(validationFailures);
}
}
public async Task ExecuteAsync(CancellationToken cancellationToken)
{
var runningTasks = new List <Task>();
while (!cancellationToken.IsCancellationRequested)
{
if (_searchParametersInitialized)
{
// Check for any changes to Search Parameters
try
{
await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken);
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
_logger.LogDebug("Reindex job worker canceled.");
}
catch (Exception ex)
{
// The job failed.
_logger.LogError(ex, "Error querying latest SearchParameterStatus updates");
}
// Check for any new Reindex Jobs
try
{
// Remove all completed tasks.
runningTasks.RemoveAll(task => task.IsCompleted);
// Get list of available jobs.
if (runningTasks.Count < _reindexJobConfiguration.MaximumNumberOfConcurrentJobsAllowed)
{
using (IScoped <IFhirOperationDataStore> store = _fhirOperationDataStoreFactory.Invoke())
{
_logger.LogTrace("Querying datastore for reindex jobs.");
IReadOnlyCollection <ReindexJobWrapper> jobs = await store.Value.AcquireReindexJobsAsync(
_reindexJobConfiguration.MaximumNumberOfConcurrentJobsAllowed,
_reindexJobConfiguration.JobHeartbeatTimeoutThreshold,
cancellationToken);
foreach (ReindexJobWrapper job in jobs)
{
_logger.LogTrace("Picked up reindex job: {jobId}.", job.JobRecord.Id);
runningTasks.Add(_reindexJobTaskFactory().ExecuteAsync(job.JobRecord, job.ETag, cancellationToken));
}
}
}
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
// End the execution of the task
}
catch (Exception ex)
{
// The job failed.
_logger.LogError(ex, "Error polling Reindex jobs.");
}
}
try
{
await Task.Delay(_reindexJobConfiguration.JobPollingFrequency, cancellationToken);
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
// End the execution of the task
}
}
}
public Transient(IScoped scoped)
{
ScopedDependency = scoped;
}
public ListSearchBehaviorTests()
{
_cancellationToken = _cancellationTokenSource.Token;
var so = new SearchOptions();
so.UnsupportedSearchParams = new Tuple <string, string> [0];
so.UnsupportedSortingParams = Array.Empty <(string searchParameterName, string reason)>();
_searchOptionsFactory = Substitute.For <ISearchOptionsFactory>();
_searchOptionsFactory.Create(Arg.Any <string>(), Arg.Any <IReadOnlyList <Tuple <string, string> > >()).Returns(so);
_fhirDataStore = Substitute.For <IFhirDataStore>();
// for an 'existing list' return a list with Patients
_fhirDataStore.GetAsync(Arg.Is <ResourceKey>(x => x.Id == "existing-list"), Arg.Any <CancellationToken>()).Returns(
x =>
{
var longList = Samples.GetDefaultList();
var rawResourceFactory = new RawResourceFactory(new FhirJsonSerializer());
return(new ResourceWrapper(
longList,
rawResourceFactory.Create(longList),
new ResourceRequest(HttpMethod.Post, "http://fhir"),
false,
null,
null,
null));
});
_scopedDataStore = Substitute.For <IScoped <IFhirDataStore> >();
_scopedDataStore.Value.Returns(_fhirDataStore);
_nonEmptyBundle = new Bundle
{
Type = Bundle.BundleType.Batch,
Entry = new List <Bundle.EntryComponent>
{
new Bundle.EntryComponent
{
Resource = Samples.GetDefaultObservation().ToPoco(),
Request = new Bundle.RequestComponent
{
Method = Bundle.HTTPVerb.POST,
Url = "Observation",
},
},
new Bundle.EntryComponent
{
Request = new Bundle.RequestComponent
{
Method = Bundle.HTTPVerb.GET,
Url = "Patient?name=peter",
},
},
},
}.ToResourceElement();
_bundleFactory = Substitute.For <IBundleFactory>();
_bundleFactory.CreateSearchBundle(Arg.Any <SearchResult>()).Returns(_nonEmptyBundle);
}
public LoggerAsyncInterceptor(IScoped scoped, ILog log)
{
this._site = scoped.GetScoped();
this._log = log;
}
private async Task RunExportCompartmentSearch(
ExportJobConfiguration exportJobConfiguration,
ExportJobProgress progress,
List <Tuple <string, string> > sharedQueryParametersList,
CancellationToken cancellationToken,
string batchIdPrefix = "")
{
EnsureArg.IsNotNull(exportJobConfiguration, nameof(exportJobConfiguration));
EnsureArg.IsNotNull(progress, nameof(progress));
EnsureArg.IsNotNull(sharedQueryParametersList, nameof(sharedQueryParametersList));
// Current batch will be used to organize a set of search results into a group so that they can be committed together.
string currentBatchId = batchIdPrefix + "-" + progress.Page.ToString("d6");
List <Tuple <string, string> > queryParametersList = new List <Tuple <string, string> >(sharedQueryParametersList);
if (progress.ContinuationToken != null)
{
queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken));
}
if (!string.IsNullOrEmpty(_exportJobRecord.ResourceType))
{
queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.Type, _exportJobRecord.ResourceType));
}
// Process the export if:
// 1. There is continuation token, which means there is more resource to be exported.
// 2. There is no continuation token but the page is 0, which means it's the initial export.
while (progress.ContinuationToken != null || progress.Page == 0)
{
SearchResult searchResult = null;
// Search and process the results.
using (IScoped <ISearchService> searchService = _searchServiceFactory())
{
searchResult = await searchService.Value.SearchCompartmentAsync(
compartmentType : KnownResourceTypes.Patient,
compartmentId : progress.TriggeringResourceId,
resourceType : null,
queryParametersList,
cancellationToken);
}
await ProcessSearchResultsAsync(searchResult.Results, currentBatchId, cancellationToken);
if (searchResult.ContinuationToken == null)
{
// No more continuation token, we are done.
break;
}
await ProcessProgressChange(exportJobConfiguration, progress, queryParametersList, searchResult.ContinuationToken, false, cancellationToken);
currentBatchId = batchIdPrefix + '-' + progress.Page.ToString("d6");
}
// Commit one last time for any pending changes.
await _exportDestinationClient.CommitAsync(exportJobConfiguration, cancellationToken);
await UpdateJobRecordAsync(cancellationToken);
}
public async Task ValidateSearchParameterInput(SearchParameter searchParam, string method, CancellationToken cancellationToken)
{
if (await _authorizationService.CheckAccess(DataActions.Reindex, cancellationToken) != DataActions.Reindex)
{
throw new UnauthorizedFhirActionException();
}
// check if reindex job is running
using (IScoped <IFhirOperationDataStore> fhirOperationDataStore = _fhirOperationDataStoreFactory())
{
(var activeReindexJobs, var reindexJobId) = await fhirOperationDataStore.Value.CheckActiveReindexJobsAsync(cancellationToken);
if (activeReindexJobs)
{
throw new JobConflictException(string.Format(Resources.ChangesToSearchParametersNotAllowedWhileReindexing, reindexJobId));
}
}
var validationFailures = new List <ValidationFailure>();
if (string.IsNullOrEmpty(searchParam.Url))
{
validationFailures.Add(
new ValidationFailure(nameof(Base.TypeName), Resources.SearchParameterDefinitionInvalidMissingUri));
}
else
{
try
{
// If a search parameter with the same uri exists already
if (_searchParameterDefinitionManager.TryGetSearchParameter(new Uri(searchParam.Url), out _))
{
// And if this is a request to create a new search parameter
if (method.Equals(HttpPostName, StringComparison.OrdinalIgnoreCase))
{
// We have a conflict
validationFailures.Add(
new ValidationFailure(
nameof(searchParam.Url),
string.Format(Resources.SearchParameterDefinitionDuplicatedEntry, searchParam.Url)));
}
else if (method.Equals(HttpPutName, StringComparison.OrdinalIgnoreCase))
{
CheckForConflictingCodeValue(searchParam, validationFailures);
}
}
else
{
// Otherwise, no search parameters with a matching uri exist
// Ensure this isn't a request to modify an existing parameter
if (method.Equals(HttpPutName, StringComparison.OrdinalIgnoreCase) ||
method.Equals(HttpDeleteName, StringComparison.OrdinalIgnoreCase))
{
validationFailures.Add(
new ValidationFailure(
nameof(searchParam.Url),
string.Format(Resources.SearchParameterDefinitionNotFound, searchParam.Url)));
}
CheckForConflictingCodeValue(searchParam, validationFailures);
}
}
catch (FormatException)
{
validationFailures.Add(
new ValidationFailure(
nameof(searchParam.Url),
string.Format(Resources.SearchParameterDefinitionInvalidDefinitionUri, searchParam.Url)));
}
}
// validate that the url does not correspond to a search param from the spec
// TODO: still need a method to determine spec defined search params
// validation of the fhir path
// TODO: separate user story for this validation
if (validationFailures.Any())
{
throw new ResourceNotValidException(validationFailures);
}
}
