private Expression CreateSearchExpression(ResourceElement coverage, ResourceElement patient)
{
var coverageValues = _searchIndexer.Extract(coverage);
var patientValues = _searchIndexer.Extract(patient);
var expressions = new List <Expression>();
var reverseChainExpressions = new List <Expression>();
expressions.Add(Expression.SearchParameter(_resourceTypeSearchParameter, Expression.StringEquals(FieldName.TokenCode, null, KnownResourceTypes.Patient, false)));
foreach (var patientValue in patientValues)
{
if (IgnoreInSearch(patientValue))
{
continue;
}
var modifier = string.Empty;
if (patientValue.SearchParameter.Type == ValueSets.SearchParamType.String)
{
modifier = ":exact";
}
expressions.Add(_expressionParser.Parse(new[] { KnownResourceTypes.Patient }, patientValue.SearchParameter.Code + modifier, patientValue.Value.ToString()));
}
foreach (var coverageValue in coverageValues)
{
if (IgnoreInSearch(coverageValue))
{
continue;
}
var modifier = string.Empty;
if (coverageValue.SearchParameter.Type == ValueSets.SearchParamType.String)
{
modifier = ":exact";
}
reverseChainExpressions.Add(_expressionParser.Parse(new[] { KnownResourceTypes.Coverage }, coverageValue.SearchParameter.Code + modifier, coverageValue.Value.ToString()));
}
if (reverseChainExpressions.Count != 0)
{
Expression reverseChainedExpression;
if (reverseChainExpressions.Count == 1)
{
reverseChainedExpression = reverseChainExpressions[0];
}
else
{
reverseChainedExpression = Expression.And(reverseChainExpressions);
}
var expression = Expression.Chained(new[] { KnownResourceTypes.Coverage }, _coverageBeneficiaryParameter, new[] { KnownResourceTypes.Patient }, true, reverseChainedExpression);
expressions.Add(expression);
}
return(Expression.And(expressions));
}
private void MockSearchIndexExtraction(string sampleName1, string sampleName2, SearchParameter searchParam)
{
SearchParameterInfo searchParamInfo = searchParam.ToInfo();
var searchIndexValues1 = new List <SearchIndexEntry>();
searchIndexValues1.Add(new SearchIndexEntry(searchParamInfo, new StringSearchValue(sampleName1)));
_searchIndexer.Extract(Arg.Is <ResourceElement>(r => r.Id.Equals(sampleName1))).Returns(searchIndexValues1);
var searchIndexValues2 = new List <SearchIndexEntry>();
searchIndexValues2.Add(new SearchIndexEntry(searchParamInfo, new StringSearchValue(sampleName2)));
_searchIndexer.Extract(Arg.Is <ResourceElement>(r => r.Id.Equals(sampleName2))).Returns(searchIndexValues2);
}
public void GivenAValidResource_WhenExtract_ThenValidSearchIndexEntriesAreCreated()
{
var coverageResource = Samples.GetDefaultCoverage().ToPoco <Coverage>();
var searchIndexEntry = _searchIndexer.Extract(coverageResource.ToResourceElement());
Assert.NotEmpty(searchIndexEntry);
var tokenSearchValue = searchIndexEntry.First().Value as TokenSearchValue;
Assert.NotNull(tokenSearchValue);
Assert.True(coverageResource.Status.Value.ToString().Equals(tokenSearchValue.Code, StringComparison.CurrentCultureIgnoreCase));
}
public async Task GivenResourcesWithUnchangedOrChangedIndices_WhenResultsProcessed_ThenCorrectResourcesHaveIndicesUpdated()
{
var searchIndexEntry1 = new SearchIndexEntry(new Core.Models.SearchParameterInfo("param1"), new StringSearchValue("value1"));
var searchIndexEntry2 = new SearchIndexEntry(new Core.Models.SearchParameterInfo("param2"), new StringSearchValue("value2"));
var searchIndices1 = new List <SearchIndexEntry>()
{
searchIndexEntry1
};
var searchIndices2 = new List <SearchIndexEntry>()
{
searchIndexEntry2
};
_searchIndexer.Extract(Arg.Any <Core.Models.ResourceElement>()).Returns(searchIndices1);
var entry1 = CreateSearchResultEntry("Patient", searchIndices1);
_output.WriteLine($"Loaded Patient with id: {entry1.Resource.ResourceId}");
var entry2 = CreateSearchResultEntry("Observation-For-Patient-f001", searchIndices2);
_output.WriteLine($"Loaded Observation with id: {entry2.Resource.ResourceId}");
var resultList = new List <SearchResultEntry>();
resultList.Add(entry1);
resultList.Add(entry2);
var result = new SearchResult(resultList, new List <Tuple <string, string> >(), new List <(string, string)>(), "token");
await _reindexUtilities.ProcessSearchResultsAsync(result, _searchParameterHashMap, CancellationToken.None);
await _fhirDataStore.Received().UpdateSearchParameterIndicesBatchAsync(
Arg.Is <IReadOnlyCollection <ResourceWrapper> >(c => c.Count() == 2), Arg.Any <CancellationToken>());
}
/// <summary>
/// For each result in a batch of resources this will extract new search params
/// Then compare those to the old values to determine if an update is needed
/// Needed updates will be committed in a batch
/// </summary>
/// <param name="results">The resource batch to process</param>
/// <param name="searchParamHash">the current hash value of the search parameters</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>A Task</returns>
public async Task ProcessSearchResultsAsync(SearchResult results, string searchParamHash, CancellationToken cancellationToken)
{
var updateHashValueOnly = new List <ResourceWrapper>();
var updateSearchIndices = new List <ResourceWrapper>();
foreach (var entry in results.Results)
{
entry.Resource.SearchParameterHash = searchParamHash;
var resourceElement = _deserializer.Deserialize(entry.Resource);
var newIndices = _searchIndexer.Extract(resourceElement);
var newIndicesHash = new HashSet <SearchIndexEntry>(newIndices);
var prevIndicesHash = new HashSet <SearchIndexEntry>(entry.Resource.SearchIndices);
if (newIndicesHash.SetEquals(prevIndicesHash))
{
updateHashValueOnly.Add(entry.Resource);
}
else
{
entry.Resource.UpdateSearchIndices(newIndices);
updateSearchIndices.Add(entry.Resource);
}
}
using (IScoped <IFhirDataStore> store = _fhirDataStoreFactory())
{
await store.Value.UpdateSearchParameterHashBatchAsync(updateHashValueOnly, cancellationToken);
await store.Value.UpdateSearchParameterIndicesBatchAsync(updateSearchIndices, cancellationToken);
}
}
/// <summary>
/// For each result in a batch of resources this will extract new search params
/// Then compare those to the old values to determine if an update is needed
/// Needed updates will be committed in a batch
/// </summary>
/// <param name="results">The resource batch to process</param>
/// <param name="resourceTypeSearchParameterHashMap">Map of resource type to current hash value of the search parameters for that resource type</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>A Task</returns>
public async Task ProcessSearchResultsAsync(SearchResult results, IReadOnlyDictionary <string, string> resourceTypeSearchParameterHashMap, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(results, nameof(results));
EnsureArg.IsNotNull(resourceTypeSearchParameterHashMap, nameof(resourceTypeSearchParameterHashMap));
var updateSearchIndices = new List <ResourceWrapper>();
foreach (var entry in results.Results)
{
if (!resourceTypeSearchParameterHashMap.TryGetValue(entry.Resource.ResourceTypeName, out string searchParamHash))
{
searchParamHash = string.Empty;
}
entry.Resource.SearchParameterHash = searchParamHash;
var resourceElement = _deserializer.Deserialize(entry.Resource);
var newIndices = _searchIndexer.Extract(resourceElement);
// TODO: If it reasonable to do so, we can compare
// old and new search indices to avoid unnecessarily updating search indices
// when not changes have been made.
entry.Resource.UpdateSearchIndices(newIndices);
updateSearchIndices.Add(entry.Resource);
if (cancellationToken.IsCancellationRequested)
{
return;
}
}
using (IScoped <IFhirDataStore> store = _fhirDataStoreFactory())
{
await store.Value.UpdateSearchParameterIndicesBatchAsync(updateSearchIndices, cancellationToken);
}
}
public async Task GivenNewSearchIndicesGetRequest_WhenHandle_ThenTheirValuesArePresentInResponse(string httpMethodName)
{
SetupDataStoreToReturnDummyResourceWrapper();
var searchIndex = new SearchIndexEntry(new SearchParameterInfo("newSearchParam", "newSearchParam"), new NumberSearchValue(1));
var searchIndex2 = new SearchIndexEntry(new SearchParameterInfo("newSearchParam2", "newSearchParam2"), new StringSearchValue("paramValue"));
var searchIndices = new List <SearchIndexEntry>()
{
searchIndex, searchIndex2
};
_searchIndexer.Extract(Arg.Any <ResourceElement>()).Returns(searchIndices);
var request = GetReindexRequest(httpMethodName);
ReindexSingleResourceResponse response = await _reindexHandler.Handle(request, _cancellationToken);
Assert.NotNull(response.ParameterResource);
Parameters parameterResponse = response.ParameterResource.ToPoco <Parameters>();
bool newSearchParamPresent = false;
bool newSearchParam2Present = false;
foreach (Parameters.ParameterComponent param in parameterResponse.Parameter)
{
if (param.Name == "newSearchParam")
{
newSearchParamPresent = true;
Assert.Equal("1", param.Value.ToString());
}
if (param.Name == "newSearchParam2")
{
newSearchParam2Present = true;
Assert.Equal("paramValue", param.Value.ToString());
}
}
Assert.True(newSearchParamPresent);
Assert.True(newSearchParam2Present);
await ValidateUpdateCallBasedOnHttpMethodType(httpMethodName);
}
/// <inheritdoc />
public ResourceWrapper Create(ResourceElement resource, bool deleted, bool keepMeta)
{
RawResource rawResource = _rawResourceFactory.Create(resource, keepMeta);
IReadOnlyCollection <SearchIndexEntry> searchIndices = _searchIndexer.Extract(resource);
string searchParamHash = _searchParameterDefinitionManager.GetSearchParameterHashForResourceType(resource.InstanceType);
ExtractMinAndMaxValues(searchIndices);
IFhirRequestContext fhirRequestContext = _fhirRequestContextAccessor.FhirRequestContext;
return(new ResourceWrapper(
resource,
rawResource,
new ResourceRequest(fhirRequestContext.Method, fhirRequestContext.Uri),
deleted,
searchIndices,
_compartmentIndexer.Extract(resource.InstanceType, searchIndices),
_claimsExtractor.Extract(),
searchParamHash));
}
public void GivenMultipleStringSearchValueForOneParameter_WhenCreate_ThenMinMaxValuesSetCorrectly()
{
var searchIndexEntry1 = new SearchIndexEntry(_nameSearchParameterInfo, new StringSearchValue("alpha"));
var searchIndexEntry2 = new SearchIndexEntry(_nameSearchParameterInfo, new StringSearchValue("beta"));
var searchIndexEntry3 = new SearchIndexEntry(_nameSearchParameterInfo, new StringSearchValue("gamma"));
_searchIndexer
.Extract(Arg.Any <ResourceElement>())
.Returns(new List <SearchIndexEntry>()
{
searchIndexEntry1, searchIndexEntry2, searchIndexEntry3
});
ResourceElement resource = Samples.GetDefaultPatient(); // Resource does not matter for this test.
ResourceWrapper resourceWrapper = _resourceWrapperFactory.Create(resource, deleted: false, keepMeta: false);
foreach (SearchIndexEntry searchEntry in resourceWrapper.SearchIndices)
{
ISupportSortSearchValue searchEntryValue = searchEntry.Value as ISupportSortSearchValue;
switch (searchEntry.Value.ToString())
{
case "alpha":
Assert.True(searchEntryValue.IsMin);
Assert.False(searchEntryValue.IsMax);
break;
case "beta":
Assert.False(searchEntryValue.IsMin);
Assert.False(searchEntryValue.IsMax);
break;
case "gamma":
Assert.False(searchEntryValue.IsMin);
Assert.True(searchEntryValue.IsMax);
break;
default:
throw new Exception("Unexpected value");
}
}
}
public void GivenAResource_WhenExtractingValues_ThenTheCorrectValuesAreReturned(string resourceFile)
{
var document = Samples.GetJsonSample <DocumentReference>(resourceFile).ToResourceElement();
var indexDocument = Samples.GetJson($"{resourceFile}.indexes");
var indexes = _indexer.Extract(document)
.Select(x => new { x.SearchParameter.Name, x.SearchParameter.Type, x.Value })
.OrderBy(x => x.Name)
.ToArray();
var asJson = JsonConvert.SerializeObject(indexes, Formatting.Indented, _settings);
Assert.Equal(indexDocument, asJson);
}
public async Task <ReindexSingleResourceResponse> Handle(ReindexSingleResourceRequest request, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(request, nameof(request));
if (await _authorizationService.CheckAccess(DataActions.Reindex) != DataActions.Reindex)
{
throw new UnauthorizedFhirActionException();
}
var key = new ResourceKey(request.ResourceType, request.ResourceId);
ResourceWrapper storedResource = await _fhirDataStore.GetAsync(key, cancellationToken);
if (storedResource == null)
{
throw new ResourceNotFoundException(string.Format(Core.Resources.ResourceNotFoundById, request.ResourceType, request.ResourceId));
}
// We need to extract the "new" search indices since the assumption is that
// a new search parameter has been added to the fhir server.
ResourceElement resourceElement = _resourceDeserializer.Deserialize(storedResource);
IReadOnlyCollection <SearchIndexEntry> newIndices = _searchIndexer.Extract(resourceElement);
// Create a new parameter resource and include the new search indices and the corresponding values.
var parametersResource = new Parameters
{
Id = Guid.NewGuid().ToString(),
VersionId = "1",
Parameter = new List <Parameters.ParameterComponent>(),
};
parametersResource.Parameter.Add(new Parameters.ParameterComponent()
{
Name = "originalResourceId", Value = new FhirString(request.ResourceId)
});
parametersResource.Parameter.Add(new Parameters.ParameterComponent()
{
Name = "originalResourceType", Value = new FhirString(request.ResourceType)
});
foreach (SearchIndexEntry searchIndex in newIndices)
{
parametersResource.Parameter.Add(new Parameters.ParameterComponent()
{
Name = searchIndex.SearchParameter.Name.ToString(), Value = new FhirString(searchIndex.Value.ToString())
});
}
return(new ReindexSingleResourceResponse(parametersResource.ToResourceElement()));
}
/// <inheritdoc />
public ResourceWrapper Create(Resource resource, bool deleted)
{
RawResource rawResource = _rawResourceFactory.Create(resource);
IReadOnlyCollection <SearchIndexEntry> searchIndices = _searchIndexer.Extract(resource);
IFhirRequestContext fhirRequestContext = _fhirRequestContextAccessor.FhirRequestContext;
return(new ResourceWrapperWithSearchIndices(
resource,
rawResource,
new ResourceRequest(fhirRequestContext.Uri, fhirRequestContext.Method),
deleted,
searchIndices,
_compartmentIndexer.Extract(resource.ResourceType, searchIndices),
_claimsIndexer.Extract()));
}
private async Task <UpsertOutcome> UpsertPatientData()
{
var json = Samples.GetJson("Patient");
var rawResource = new RawResource(json, FhirResourceFormat.Json, isMetaSet: false);
var resourceRequest = new ResourceRequest(WebRequestMethods.Http.Put);
var compartmentIndices = Substitute.For <CompartmentIndices>();
var resourceElement = Deserializers.ResourceDeserializer.DeserializeRaw(rawResource, "v1", DateTimeOffset.UtcNow);
var searchIndices = _searchIndexer.Extract(resourceElement);
var wrapper = new ResourceWrapper(
resourceElement,
rawResource,
resourceRequest,
false,
searchIndices,
compartmentIndices,
new List <KeyValuePair <string, string> >(),
_searchParameterDefinitionManager.GetSearchParameterHashForResourceType("Patient"));
return(await _scopedDataStore.Value.UpsertAsync(wrapper, null, true, true, CancellationToken.None));
}
public async System.Threading.Tasks.Task GivenResourcesWithUnchangedOrChangedIndices_WhenResultsProcessed_ThenCorrectResourcesHaveIndicesUpdated()
{
Func <Health.Extensions.DependencyInjection.IScoped <IFhirDataStore> > fhirDataStoreScope = () => _fhirDataStore.CreateMockScope();
var utilities = new ReindexUtilities(fhirDataStoreScope, _searchIndexer, _resourceDeserializer);
var searchIndices1 = new List <SearchIndexEntry>()
{
new SearchIndexEntry(new Core.Models.SearchParameterInfo("param1"), new StringSearchValue("value1"))
};
var searchIndices2 = new List <SearchIndexEntry>()
{
new SearchIndexEntry(new Core.Models.SearchParameterInfo("param2"), new StringSearchValue("value2"))
};
_searchIndexer.Extract(Arg.Any <Core.Models.ResourceElement>()).Returns(searchIndices1);
var entry1 = CreateSearchResultEntry("Patient", searchIndices1);
_output.WriteLine($"Loaded Patient with id: {entry1.Resource.ResourceId}");
var entry2 = CreateSearchResultEntry("Observation-For-Patient-f001", searchIndices2);
_output.WriteLine($"Loaded Observation with id: {entry2.Resource.ResourceId}");
var resultList = new List <SearchResultEntry>();
resultList.Add(entry1);
resultList.Add(entry2);
var result = new SearchResult(resultList, new List <Tuple <string, string> >(), new List <(string, string)>(), "token");
await utilities.ProcessSearchResultsAsync(result, "hash", CancellationToken.None);
await _fhirDataStore.Received().UpdateSearchParameterHashBatchAsync(
Arg.Is <IReadOnlyCollection <ResourceWrapper> >(
c => c.Where(r => r.SearchIndices == searchIndices1 && r.ResourceTypeName.Equals("Patient")).Count() == 1),
Arg.Any <CancellationToken>());
await _fhirDataStore.Received().UpdateSearchParameterIndicesBatchAsync(
Arg.Is <IReadOnlyCollection <ResourceWrapper> >(
c => c.Where(r => r.SearchIndices == searchIndices1 && r.ResourceTypeName.Equals("Observation")).Count() == 1),
Arg.Any <CancellationToken>());
}
public void GivenAResource_WhenExtractingValues_ThenTheCorrectValuesAreReturned(string resourceFile)
{
var document = Samples.GetJsonSample <DocumentReference>(resourceFile).ToResourceElement();
var indexDocument = Samples.GetJson($"{resourceFile}.indexes");
var indices = _indexer.Extract(document)
.Select(x => new { x.SearchParameter.Name, x.SearchParameter.Type, x.Value })
.OrderBy(x => x.Name)
.ToArray();
var extractedIndices = new List <JToken>();
foreach (var index in indices)
{
extractedIndices.Add(JToken.Parse(JsonConvert.SerializeObject(index, Formatting.Indented, _settings)));
}
var expectedJObjects = JArray.Parse(indexDocument);
for (var i = 0; i < expectedJObjects.Count; i++)
{
Assert.True(JToken.DeepEquals(expectedJObjects[i], extractedIndices[i]));
}
}
public async Task GivenNewSearchParam_WhenReindexJobCompleted_ThenParamIsSearchable()
{
var searchParamName = "foo";
var searchParamCode = "fooCode";
var searchParam = new SearchParameterInfo(
name: searchParamName,
code: searchParamCode,
searchParamType: ValueSets.SearchParamType.String,
url: new Uri("http://hl7.org/fhir/SearchParameter/Patient-foo"),
components: null,
expression: "Patient.name",
targetResourceTypes: null,
baseResourceTypes: new List <string>()
{
"Patient"
})
{
IsSupported = true,
IsSearchable = false,
};
_searchParameterDefinitionManager.UrlLookup.TryAdd(searchParam.Url, searchParam);
_searchParameterDefinitionManager.TypeLookup["Patient"].TryAdd(searchParamCode, searchParam);
await UpsertPatientData("searchIndicesPatient1");
await UpsertPatientData("searchIndicesPatient2");
var queryParams = new List <Tuple <string, string> >()
{
new Tuple <string, string>("fooCode", "searchIndicesPatient1")
};
var searchResults = await _searchService.Value.SearchAsync("Patient", queryParams, CancellationToken.None);
Assert.Equal(searchParamCode, searchResults.UnsupportedSearchParameters.FirstOrDefault().Item1);
Assert.Equal(2, searchResults.Results.Count());
var searchIndexValues1 = new List <SearchIndexEntry>();
searchIndexValues1.Add(new SearchIndexEntry(searchParam, new StringSearchValue("searchIndicesPatient1")));
_searchIndexer.Extract(Arg.Is <ResourceElement>(r => r.Id.Equals("searchIndicesPatient1"))).Returns(searchIndexValues1);
var searchIndexValues2 = new List <SearchIndexEntry>();
searchIndexValues2.Add(new SearchIndexEntry(searchParam, new StringSearchValue("searchIndicesPatient2")));
_searchIndexer.Extract(Arg.Is <ResourceElement>(r => r.Id.Equals("searchIndicesPatient2"))).Returns(searchIndexValues2);
var request = new CreateReindexRequest();
CreateReindexResponse response = await _createReindexRequestHandler.Handle(request, CancellationToken.None);
Assert.NotNull(response);
Assert.False(string.IsNullOrWhiteSpace(response.Job.JobRecord.Id));
_reindexJobWorker = new ReindexJobWorker(
() => _scopedOperationDataStore,
Options.Create(_jobConfiguration),
InitializeReindexJobTask,
NullLogger <ReindexJobWorker> .Instance);
var cancellationTokenSource = new CancellationTokenSource();
try
{
var reindexWorkerTask = _reindexJobWorker.ExecuteAsync(cancellationTokenSource.Token);
var reindexJobWrapper = await _fhirOperationDataStore.GetReindexJobByIdAsync(response.Job.JobRecord.Id, cancellationTokenSource.Token);
int delayCount = 0;
while (reindexJobWrapper.JobRecord.Status != OperationStatus.Completed &&
delayCount < 10)
{
await Task.Delay(1000);
delayCount++;
reindexJobWrapper = await _fhirOperationDataStore.GetReindexJobByIdAsync(response.Job.JobRecord.Id, cancellationTokenSource.Token);
}
Assert.True(delayCount <= 9);
searchResults = await _searchService.Value.SearchAsync("Patient", queryParams, CancellationToken.None);
Assert.Single(searchResults.Results);
var patient = searchResults.Results.FirstOrDefault().Resource;
Assert.Contains("searchIndicesPatient1", patient.RawResource.Data);
}
finally
{
cancellationTokenSource.Cancel();
}
}