private Expression CreateSearchExpression(ResourceElement coverage, ResourceElement patient) { var coverageValues = _searchIndexer.Extract(coverage); var patientValues = _searchIndexer.Extract(patient); var expressions = new List <Expression>(); var reverseChainExpressions = new List <Expression>(); expressions.Add(Expression.SearchParameter(_resourceTypeSearchParameter, Expression.StringEquals(FieldName.TokenCode, null, KnownResourceTypes.Patient, false))); foreach (var patientValue in patientValues) { if (IgnoreInSearch(patientValue)) { continue; } var modifier = string.Empty; if (patientValue.SearchParameter.Type == ValueSets.SearchParamType.String) { modifier = ":exact"; } expressions.Add(_expressionParser.Parse(new[] { KnownResourceTypes.Patient }, patientValue.SearchParameter.Code + modifier, patientValue.Value.ToString())); } foreach (var coverageValue in coverageValues) { if (IgnoreInSearch(coverageValue)) { continue; } var modifier = string.Empty; if (coverageValue.SearchParameter.Type == ValueSets.SearchParamType.String) { modifier = ":exact"; } reverseChainExpressions.Add(_expressionParser.Parse(new[] { KnownResourceTypes.Coverage }, coverageValue.SearchParameter.Code + modifier, coverageValue.Value.ToString())); } if (reverseChainExpressions.Count != 0) { Expression reverseChainedExpression; if (reverseChainExpressions.Count == 1) { reverseChainedExpression = reverseChainExpressions[0]; } else { reverseChainedExpression = Expression.And(reverseChainExpressions); } var expression = Expression.Chained(new[] { KnownResourceTypes.Coverage }, _coverageBeneficiaryParameter, new[] { KnownResourceTypes.Patient }, true, reverseChainedExpression); expressions.Add(expression); } return(Expression.And(expressions)); }
private void MockSearchIndexExtraction(string sampleName1, string sampleName2, SearchParameter searchParam) { SearchParameterInfo searchParamInfo = searchParam.ToInfo(); var searchIndexValues1 = new List <SearchIndexEntry>(); searchIndexValues1.Add(new SearchIndexEntry(searchParamInfo, new StringSearchValue(sampleName1))); _searchIndexer.Extract(Arg.Is <ResourceElement>(r => r.Id.Equals(sampleName1))).Returns(searchIndexValues1); var searchIndexValues2 = new List <SearchIndexEntry>(); searchIndexValues2.Add(new SearchIndexEntry(searchParamInfo, new StringSearchValue(sampleName2))); _searchIndexer.Extract(Arg.Is <ResourceElement>(r => r.Id.Equals(sampleName2))).Returns(searchIndexValues2); }
public void GivenAValidResource_WhenExtract_ThenValidSearchIndexEntriesAreCreated() { var coverageResource = Samples.GetDefaultCoverage().ToPoco <Coverage>(); var searchIndexEntry = _searchIndexer.Extract(coverageResource.ToResourceElement()); Assert.NotEmpty(searchIndexEntry); var tokenSearchValue = searchIndexEntry.First().Value as TokenSearchValue; Assert.NotNull(tokenSearchValue); Assert.True(coverageResource.Status.Value.ToString().Equals(tokenSearchValue.Code, StringComparison.CurrentCultureIgnoreCase)); }
public async Task GivenResourcesWithUnchangedOrChangedIndices_WhenResultsProcessed_ThenCorrectResourcesHaveIndicesUpdated() { var searchIndexEntry1 = new SearchIndexEntry(new Core.Models.SearchParameterInfo("param1"), new StringSearchValue("value1")); var searchIndexEntry2 = new SearchIndexEntry(new Core.Models.SearchParameterInfo("param2"), new StringSearchValue("value2")); var searchIndices1 = new List <SearchIndexEntry>() { searchIndexEntry1 }; var searchIndices2 = new List <SearchIndexEntry>() { searchIndexEntry2 }; _searchIndexer.Extract(Arg.Any <Core.Models.ResourceElement>()).Returns(searchIndices1); var entry1 = CreateSearchResultEntry("Patient", searchIndices1); _output.WriteLine($"Loaded Patient with id: {entry1.Resource.ResourceId}"); var entry2 = CreateSearchResultEntry("Observation-For-Patient-f001", searchIndices2); _output.WriteLine($"Loaded Observation with id: {entry2.Resource.ResourceId}"); var resultList = new List <SearchResultEntry>(); resultList.Add(entry1); resultList.Add(entry2); var result = new SearchResult(resultList, new List <Tuple <string, string> >(), new List <(string, string)>(), "token"); await _reindexUtilities.ProcessSearchResultsAsync(result, _searchParameterHashMap, CancellationToken.None); await _fhirDataStore.Received().UpdateSearchParameterIndicesBatchAsync( Arg.Is <IReadOnlyCollection <ResourceWrapper> >(c => c.Count() == 2), Arg.Any <CancellationToken>()); }
/// <summary> /// For each result in a batch of resources this will extract new search params /// Then compare those to the old values to determine if an update is needed /// Needed updates will be committed in a batch /// </summary> /// <param name="results">The resource batch to process</param> /// <param name="searchParamHash">the current hash value of the search parameters</param> /// <param name="cancellationToken">Cancellation token</param> /// <returns>A Task</returns> public async Task ProcessSearchResultsAsync(SearchResult results, string searchParamHash, CancellationToken cancellationToken) { var updateHashValueOnly = new List <ResourceWrapper>(); var updateSearchIndices = new List <ResourceWrapper>(); foreach (var entry in results.Results) { entry.Resource.SearchParameterHash = searchParamHash; var resourceElement = _deserializer.Deserialize(entry.Resource); var newIndices = _searchIndexer.Extract(resourceElement); var newIndicesHash = new HashSet <SearchIndexEntry>(newIndices); var prevIndicesHash = new HashSet <SearchIndexEntry>(entry.Resource.SearchIndices); if (newIndicesHash.SetEquals(prevIndicesHash)) { updateHashValueOnly.Add(entry.Resource); } else { entry.Resource.UpdateSearchIndices(newIndices); updateSearchIndices.Add(entry.Resource); } } using (IScoped <IFhirDataStore> store = _fhirDataStoreFactory()) { await store.Value.UpdateSearchParameterHashBatchAsync(updateHashValueOnly, cancellationToken); await store.Value.UpdateSearchParameterIndicesBatchAsync(updateSearchIndices, cancellationToken); } }
/// <summary> /// For each result in a batch of resources this will extract new search params /// Then compare those to the old values to determine if an update is needed /// Needed updates will be committed in a batch /// </summary> /// <param name="results">The resource batch to process</param> /// <param name="resourceTypeSearchParameterHashMap">Map of resource type to current hash value of the search parameters for that resource type</param> /// <param name="cancellationToken">Cancellation token</param> /// <returns>A Task</returns> public async Task ProcessSearchResultsAsync(SearchResult results, IReadOnlyDictionary <string, string> resourceTypeSearchParameterHashMap, CancellationToken cancellationToken) { EnsureArg.IsNotNull(results, nameof(results)); EnsureArg.IsNotNull(resourceTypeSearchParameterHashMap, nameof(resourceTypeSearchParameterHashMap)); var updateSearchIndices = new List <ResourceWrapper>(); foreach (var entry in results.Results) { if (!resourceTypeSearchParameterHashMap.TryGetValue(entry.Resource.ResourceTypeName, out string searchParamHash)) { searchParamHash = string.Empty; } entry.Resource.SearchParameterHash = searchParamHash; var resourceElement = _deserializer.Deserialize(entry.Resource); var newIndices = _searchIndexer.Extract(resourceElement); // TODO: If it reasonable to do so, we can compare // old and new search indices to avoid unnecessarily updating search indices // when not changes have been made. entry.Resource.UpdateSearchIndices(newIndices); updateSearchIndices.Add(entry.Resource); if (cancellationToken.IsCancellationRequested) { return; } } using (IScoped <IFhirDataStore> store = _fhirDataStoreFactory()) { await store.Value.UpdateSearchParameterIndicesBatchAsync(updateSearchIndices, cancellationToken); } }
public async Task GivenNewSearchIndicesGetRequest_WhenHandle_ThenTheirValuesArePresentInResponse(string httpMethodName) { SetupDataStoreToReturnDummyResourceWrapper(); var searchIndex = new SearchIndexEntry(new SearchParameterInfo("newSearchParam", "newSearchParam"), new NumberSearchValue(1)); var searchIndex2 = new SearchIndexEntry(new SearchParameterInfo("newSearchParam2", "newSearchParam2"), new StringSearchValue("paramValue")); var searchIndices = new List <SearchIndexEntry>() { searchIndex, searchIndex2 }; _searchIndexer.Extract(Arg.Any <ResourceElement>()).Returns(searchIndices); var request = GetReindexRequest(httpMethodName); ReindexSingleResourceResponse response = await _reindexHandler.Handle(request, _cancellationToken); Assert.NotNull(response.ParameterResource); Parameters parameterResponse = response.ParameterResource.ToPoco <Parameters>(); bool newSearchParamPresent = false; bool newSearchParam2Present = false; foreach (Parameters.ParameterComponent param in parameterResponse.Parameter) { if (param.Name == "newSearchParam") { newSearchParamPresent = true; Assert.Equal("1", param.Value.ToString()); } if (param.Name == "newSearchParam2") { newSearchParam2Present = true; Assert.Equal("paramValue", param.Value.ToString()); } } Assert.True(newSearchParamPresent); Assert.True(newSearchParam2Present); await ValidateUpdateCallBasedOnHttpMethodType(httpMethodName); }
/// <inheritdoc /> public ResourceWrapper Create(ResourceElement resource, bool deleted, bool keepMeta) { RawResource rawResource = _rawResourceFactory.Create(resource, keepMeta); IReadOnlyCollection <SearchIndexEntry> searchIndices = _searchIndexer.Extract(resource); string searchParamHash = _searchParameterDefinitionManager.GetSearchParameterHashForResourceType(resource.InstanceType); ExtractMinAndMaxValues(searchIndices); IFhirRequestContext fhirRequestContext = _fhirRequestContextAccessor.FhirRequestContext; return(new ResourceWrapper( resource, rawResource, new ResourceRequest(fhirRequestContext.Method, fhirRequestContext.Uri), deleted, searchIndices, _compartmentIndexer.Extract(resource.InstanceType, searchIndices), _claimsExtractor.Extract(), searchParamHash)); }
public void GivenMultipleStringSearchValueForOneParameter_WhenCreate_ThenMinMaxValuesSetCorrectly() { var searchIndexEntry1 = new SearchIndexEntry(_nameSearchParameterInfo, new StringSearchValue("alpha")); var searchIndexEntry2 = new SearchIndexEntry(_nameSearchParameterInfo, new StringSearchValue("beta")); var searchIndexEntry3 = new SearchIndexEntry(_nameSearchParameterInfo, new StringSearchValue("gamma")); _searchIndexer .Extract(Arg.Any <ResourceElement>()) .Returns(new List <SearchIndexEntry>() { searchIndexEntry1, searchIndexEntry2, searchIndexEntry3 }); ResourceElement resource = Samples.GetDefaultPatient(); // Resource does not matter for this test. ResourceWrapper resourceWrapper = _resourceWrapperFactory.Create(resource, deleted: false, keepMeta: false); foreach (SearchIndexEntry searchEntry in resourceWrapper.SearchIndices) { ISupportSortSearchValue searchEntryValue = searchEntry.Value as ISupportSortSearchValue; switch (searchEntry.Value.ToString()) { case "alpha": Assert.True(searchEntryValue.IsMin); Assert.False(searchEntryValue.IsMax); break; case "beta": Assert.False(searchEntryValue.IsMin); Assert.False(searchEntryValue.IsMax); break; case "gamma": Assert.False(searchEntryValue.IsMin); Assert.True(searchEntryValue.IsMax); break; default: throw new Exception("Unexpected value"); } } }
public void GivenAResource_WhenExtractingValues_ThenTheCorrectValuesAreReturned(string resourceFile) { var document = Samples.GetJsonSample <DocumentReference>(resourceFile).ToResourceElement(); var indexDocument = Samples.GetJson($"{resourceFile}.indexes"); var indexes = _indexer.Extract(document) .Select(x => new { x.SearchParameter.Name, x.SearchParameter.Type, x.Value }) .OrderBy(x => x.Name) .ToArray(); var asJson = JsonConvert.SerializeObject(indexes, Formatting.Indented, _settings); Assert.Equal(indexDocument, asJson); }
public async Task <ReindexSingleResourceResponse> Handle(ReindexSingleResourceRequest request, CancellationToken cancellationToken) { EnsureArg.IsNotNull(request, nameof(request)); if (await _authorizationService.CheckAccess(DataActions.Reindex) != DataActions.Reindex) { throw new UnauthorizedFhirActionException(); } var key = new ResourceKey(request.ResourceType, request.ResourceId); ResourceWrapper storedResource = await _fhirDataStore.GetAsync(key, cancellationToken); if (storedResource == null) { throw new ResourceNotFoundException(string.Format(Core.Resources.ResourceNotFoundById, request.ResourceType, request.ResourceId)); } // We need to extract the "new" search indices since the assumption is that // a new search parameter has been added to the fhir server. ResourceElement resourceElement = _resourceDeserializer.Deserialize(storedResource); IReadOnlyCollection <SearchIndexEntry> newIndices = _searchIndexer.Extract(resourceElement); // Create a new parameter resource and include the new search indices and the corresponding values. var parametersResource = new Parameters { Id = Guid.NewGuid().ToString(), VersionId = "1", Parameter = new List <Parameters.ParameterComponent>(), }; parametersResource.Parameter.Add(new Parameters.ParameterComponent() { Name = "originalResourceId", Value = new FhirString(request.ResourceId) }); parametersResource.Parameter.Add(new Parameters.ParameterComponent() { Name = "originalResourceType", Value = new FhirString(request.ResourceType) }); foreach (SearchIndexEntry searchIndex in newIndices) { parametersResource.Parameter.Add(new Parameters.ParameterComponent() { Name = searchIndex.SearchParameter.Name.ToString(), Value = new FhirString(searchIndex.Value.ToString()) }); } return(new ReindexSingleResourceResponse(parametersResource.ToResourceElement())); }
/// <inheritdoc /> public ResourceWrapper Create(Resource resource, bool deleted) { RawResource rawResource = _rawResourceFactory.Create(resource); IReadOnlyCollection <SearchIndexEntry> searchIndices = _searchIndexer.Extract(resource); IFhirRequestContext fhirRequestContext = _fhirRequestContextAccessor.FhirRequestContext; return(new ResourceWrapperWithSearchIndices( resource, rawResource, new ResourceRequest(fhirRequestContext.Uri, fhirRequestContext.Method), deleted, searchIndices, _compartmentIndexer.Extract(resource.ResourceType, searchIndices), _claimsIndexer.Extract())); }
private async Task <UpsertOutcome> UpsertPatientData() { var json = Samples.GetJson("Patient"); var rawResource = new RawResource(json, FhirResourceFormat.Json, isMetaSet: false); var resourceRequest = new ResourceRequest(WebRequestMethods.Http.Put); var compartmentIndices = Substitute.For <CompartmentIndices>(); var resourceElement = Deserializers.ResourceDeserializer.DeserializeRaw(rawResource, "v1", DateTimeOffset.UtcNow); var searchIndices = _searchIndexer.Extract(resourceElement); var wrapper = new ResourceWrapper( resourceElement, rawResource, resourceRequest, false, searchIndices, compartmentIndices, new List <KeyValuePair <string, string> >(), _searchParameterDefinitionManager.GetSearchParameterHashForResourceType("Patient")); return(await _scopedDataStore.Value.UpsertAsync(wrapper, null, true, true, CancellationToken.None)); }
public async System.Threading.Tasks.Task GivenResourcesWithUnchangedOrChangedIndices_WhenResultsProcessed_ThenCorrectResourcesHaveIndicesUpdated() { Func <Health.Extensions.DependencyInjection.IScoped <IFhirDataStore> > fhirDataStoreScope = () => _fhirDataStore.CreateMockScope(); var utilities = new ReindexUtilities(fhirDataStoreScope, _searchIndexer, _resourceDeserializer); var searchIndices1 = new List <SearchIndexEntry>() { new SearchIndexEntry(new Core.Models.SearchParameterInfo("param1"), new StringSearchValue("value1")) }; var searchIndices2 = new List <SearchIndexEntry>() { new SearchIndexEntry(new Core.Models.SearchParameterInfo("param2"), new StringSearchValue("value2")) }; _searchIndexer.Extract(Arg.Any <Core.Models.ResourceElement>()).Returns(searchIndices1); var entry1 = CreateSearchResultEntry("Patient", searchIndices1); _output.WriteLine($"Loaded Patient with id: {entry1.Resource.ResourceId}"); var entry2 = CreateSearchResultEntry("Observation-For-Patient-f001", searchIndices2); _output.WriteLine($"Loaded Observation with id: {entry2.Resource.ResourceId}"); var resultList = new List <SearchResultEntry>(); resultList.Add(entry1); resultList.Add(entry2); var result = new SearchResult(resultList, new List <Tuple <string, string> >(), new List <(string, string)>(), "token"); await utilities.ProcessSearchResultsAsync(result, "hash", CancellationToken.None); await _fhirDataStore.Received().UpdateSearchParameterHashBatchAsync( Arg.Is <IReadOnlyCollection <ResourceWrapper> >( c => c.Where(r => r.SearchIndices == searchIndices1 && r.ResourceTypeName.Equals("Patient")).Count() == 1), Arg.Any <CancellationToken>()); await _fhirDataStore.Received().UpdateSearchParameterIndicesBatchAsync( Arg.Is <IReadOnlyCollection <ResourceWrapper> >( c => c.Where(r => r.SearchIndices == searchIndices1 && r.ResourceTypeName.Equals("Observation")).Count() == 1), Arg.Any <CancellationToken>()); }
public void GivenAResource_WhenExtractingValues_ThenTheCorrectValuesAreReturned(string resourceFile) { var document = Samples.GetJsonSample <DocumentReference>(resourceFile).ToResourceElement(); var indexDocument = Samples.GetJson($"{resourceFile}.indexes"); var indices = _indexer.Extract(document) .Select(x => new { x.SearchParameter.Name, x.SearchParameter.Type, x.Value }) .OrderBy(x => x.Name) .ToArray(); var extractedIndices = new List <JToken>(); foreach (var index in indices) { extractedIndices.Add(JToken.Parse(JsonConvert.SerializeObject(index, Formatting.Indented, _settings))); } var expectedJObjects = JArray.Parse(indexDocument); for (var i = 0; i < expectedJObjects.Count; i++) { Assert.True(JToken.DeepEquals(expectedJObjects[i], extractedIndices[i])); } }
public async Task GivenNewSearchParam_WhenReindexJobCompleted_ThenParamIsSearchable() { var searchParamName = "foo"; var searchParamCode = "fooCode"; var searchParam = new SearchParameterInfo( name: searchParamName, code: searchParamCode, searchParamType: ValueSets.SearchParamType.String, url: new Uri("http://hl7.org/fhir/SearchParameter/Patient-foo"), components: null, expression: "Patient.name", targetResourceTypes: null, baseResourceTypes: new List <string>() { "Patient" }) { IsSupported = true, IsSearchable = false, }; _searchParameterDefinitionManager.UrlLookup.TryAdd(searchParam.Url, searchParam); _searchParameterDefinitionManager.TypeLookup["Patient"].TryAdd(searchParamCode, searchParam); await UpsertPatientData("searchIndicesPatient1"); await UpsertPatientData("searchIndicesPatient2"); var queryParams = new List <Tuple <string, string> >() { new Tuple <string, string>("fooCode", "searchIndicesPatient1") }; var searchResults = await _searchService.Value.SearchAsync("Patient", queryParams, CancellationToken.None); Assert.Equal(searchParamCode, searchResults.UnsupportedSearchParameters.FirstOrDefault().Item1); Assert.Equal(2, searchResults.Results.Count()); var searchIndexValues1 = new List <SearchIndexEntry>(); searchIndexValues1.Add(new SearchIndexEntry(searchParam, new StringSearchValue("searchIndicesPatient1"))); _searchIndexer.Extract(Arg.Is <ResourceElement>(r => r.Id.Equals("searchIndicesPatient1"))).Returns(searchIndexValues1); var searchIndexValues2 = new List <SearchIndexEntry>(); searchIndexValues2.Add(new SearchIndexEntry(searchParam, new StringSearchValue("searchIndicesPatient2"))); _searchIndexer.Extract(Arg.Is <ResourceElement>(r => r.Id.Equals("searchIndicesPatient2"))).Returns(searchIndexValues2); var request = new CreateReindexRequest(); CreateReindexResponse response = await _createReindexRequestHandler.Handle(request, CancellationToken.None); Assert.NotNull(response); Assert.False(string.IsNullOrWhiteSpace(response.Job.JobRecord.Id)); _reindexJobWorker = new ReindexJobWorker( () => _scopedOperationDataStore, Options.Create(_jobConfiguration), InitializeReindexJobTask, NullLogger <ReindexJobWorker> .Instance); var cancellationTokenSource = new CancellationTokenSource(); try { var reindexWorkerTask = _reindexJobWorker.ExecuteAsync(cancellationTokenSource.Token); var reindexJobWrapper = await _fhirOperationDataStore.GetReindexJobByIdAsync(response.Job.JobRecord.Id, cancellationTokenSource.Token); int delayCount = 0; while (reindexJobWrapper.JobRecord.Status != OperationStatus.Completed && delayCount < 10) { await Task.Delay(1000); delayCount++; reindexJobWrapper = await _fhirOperationDataStore.GetReindexJobByIdAsync(response.Job.JobRecord.Id, cancellationTokenSource.Token); } Assert.True(delayCount <= 9); searchResults = await _searchService.Value.SearchAsync("Patient", queryParams, CancellationToken.None); Assert.Single(searchResults.Results); var patient = searchResults.Results.FirstOrDefault().Resource; Assert.Contains("searchIndicesPatient1", patient.RawResource.Data); } finally { cancellationTokenSource.Cancel(); } }