public async Task GivenListOfResourcesWithDupResourceId_WhenBulkMergeToStore_ThenDistinctResourceShouldBeImported()
{
List <SqlBulkCopyDataWrapper> resources = new List <SqlBulkCopyDataWrapper>();
long startSurrogateId = ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(DateTime.Now);
SqlBulkCopyDataWrapper resource1 = CreateTestResource(Guid.NewGuid().ToString(), startSurrogateId);
SqlBulkCopyDataWrapper resource2 = CreateTestResource(Guid.NewGuid().ToString(), startSurrogateId + 1);
resources.Add(resource1);
resources.Add(resource2);
SqlBulkCopyDataWrapper[] result = (await _sqlServerFhirDataBulkOperation.BulkMergeResourceAsync(resources, CancellationToken.None)).ToArray();
int rCount = await GetResourceCountAsync("Resource", startSurrogateId, startSurrogateId + 2);
Assert.Equal(2, result.Count());
Assert.Equal(2, rCount);
resource1.ResourceSurrogateId = startSurrogateId + 2;
resource1.ResourceSurrogateId = startSurrogateId + 3;
result = (await _sqlServerFhirDataBulkOperation.BulkMergeResourceAsync(resources, CancellationToken.None)).ToArray();
rCount = await GetResourceCountAsync("Resource", startSurrogateId, startSurrogateId + 4);
Assert.Empty(result);
Assert.Equal(2, rCount);
}
public async Task GivenImportedBatchResources_WhenCleanData_ThenRecordsShouldBeDeleted()
{
long startSurrogateId = ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(DateTime.Now);
int count = 1001;
short typeId = _fixture.SqlServerFhirModel.GetResourceTypeId("Patient");
List <string> tableNames = new List <string>();
tableNames.Add(await ImportDataAsync(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateResourceTable));
tableNames.Add(await ImportDataAsync(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateDateTimeSearchParamsTable));
tableNames.Add(await ImportDataAsync(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateNumberSearchParamsTable));
tableNames.Add(await ImportDataAsync(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateQuantitySearchParamsTable));
tableNames.Add(await ImportDataAsync(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateReferenceSearchParamsTable));
tableNames.Add(await ImportDataAsync(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateReferenceTokenCompositeSearchParamsTable));
tableNames.Add(await ImportDataAsync(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateStringSearchParamsTable));
tableNames.Add(await ImportDataAsync(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateTokenDateTimeCompositeSearchParamsTable));
tableNames.Add(await ImportDataAsync(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateTokenNumberNumberCompositeSearchParamsTable));
tableNames.Add(await ImportDataAsync(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateTokenQuantityCompositeSearchParamsTable));
tableNames.Add(await ImportDataAsync(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateTokenSearchParamsTable));
tableNames.Add(await ImportDataAsync(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateTokenStringCompositeSearchParamsTable));
tableNames.Add(await ImportDataAsync(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateTokenTextSearchParamsTable));
tableNames.Add(await ImportDataAsync(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateTokenTokenCompositeSearchParamsTable));
tableNames.Add(await ImportDataAsync(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateUriSearchParamsTable));
tableNames.Add(await ImportDataAsync(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateCompartmentAssignmentTable));
tableNames.Add(await ImportDataAsync(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateResourceWriteClaimTable));
await _sqlServerFhirDataBulkOperation.CleanBatchResourceAsync("Patient", startSurrogateId, startSurrogateId + count - 1, CancellationToken.None);
foreach (string tableName in tableNames)
{
int rCount = await GetResourceCountAsync(tableName, startSurrogateId, startSurrogateId + count);
Assert.Equal(1, rCount);
}
}
public async Task GivenBatchInValidResources_WhenBulkCopy_ThenExceptionShouldBeThrow()
{
long startSurrogateId = ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(DateTime.Now);
int count = 1001;
DataTable inputTable = TestBulkDataProvider.GenerateInValidUriSearchParamsTable(count, startSurrogateId, 0);
await Assert.ThrowsAnyAsync <Exception>(async() => await _sqlServerFhirDataBulkOperation.BulkCopyDataAsync(inputTable, CancellationToken.None));
}
public void Populate(long id, Resource resource)
{
if (resource.Meta == null)
{
resource.Meta = new Meta();
}
resource.Meta.LastUpdated = ResourceSurrogateIdHelper.ResourceSurrogateIdToLastUpdated(id);
}
public override Expression VisitBinary(BinaryExpression expression, object context)
{
if (expression.FieldName != FieldName.DateTimeStart && expression.FieldName != FieldName.DateTimeEnd)
{
throw new ArgumentOutOfRangeException(expression.FieldName.ToString());
}
// ResourceSurrogateId has millisecond datetime precision, with lower bits added in to make the value unique.
DateTime original = ((DateTimeOffset)expression.Value).UtcDateTime;
DateTime truncated = original.TruncateToMillisecond();
switch (expression.BinaryOperator)
{
case BinaryOperator.GreaterThan:
return(Expression.GreaterThanOrEqual(
SqlFieldName.ResourceSurrogateId,
null,
ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(truncated.AddTicks(TimeSpan.TicksPerMillisecond))));
case BinaryOperator.GreaterThanOrEqual:
if (original == truncated)
{
return(Expression.GreaterThanOrEqual(
SqlFieldName.ResourceSurrogateId,
null,
ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(truncated)));
}
goto case BinaryOperator.GreaterThan;
case BinaryOperator.LessThan:
if (original == truncated)
{
return(Expression.LessThan(
SqlFieldName.ResourceSurrogateId,
null,
ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(truncated)));
}
goto case BinaryOperator.LessThanOrEqual;
case BinaryOperator.LessThanOrEqual:
return(Expression.LessThan(
SqlFieldName.ResourceSurrogateId,
null,
ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(truncated.AddTicks(TimeSpan.TicksPerMillisecond))));
case BinaryOperator.NotEqual:
case BinaryOperator.Equal: // expecting eq to have been rewritten as a range
default:
throw new ArgumentOutOfRangeException(expression.BinaryOperator.ToString());
}
}
public void GivenAnExpressionOverLastUpdated_WhenTranslatedToResourceSurrogateId_HasCorrectRanges(BinaryOperator inputOperator, string inputDateTimeOffset, BinaryOperator expectedOperator, string expectedDateTimeOffset)
{
var input = new BinaryExpression(inputOperator, FieldName.DateTimeStart, null, DateTimeOffset.Parse(inputDateTimeOffset));
var output = input.AcceptVisitor(LastUpdatedToResourceSurrogateIdRewriter.Instance, null);
BinaryExpression binaryOutput = Assert.IsType <BinaryExpression>(output);
Assert.Equal(SqlFieldName.ResourceSurrogateId, binaryOutput.FieldName);
Assert.Equal(expectedOperator, binaryOutput.BinaryOperator);
Assert.Equal(DateTimeOffset.Parse(expectedDateTimeOffset), ResourceSurrogateIdHelper.ResourceSurrogateIdToLastUpdated((long)binaryOutput.Value));
}
public void GivenADateTime_WhenRepresentedAsASurrogateId_HasTheExpectedRange()
{
var baseDate = DateTime.MinValue;
long baseId = ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(baseDate);
Assert.Equal(baseDate, ResourceSurrogateIdHelper.ResourceSurrogateIdToLastUpdated(baseId + 79999));
Assert.Equal(TimeSpan.FromTicks(TimeSpan.TicksPerMillisecond), ResourceSurrogateIdHelper.ResourceSurrogateIdToLastUpdated(baseId + 80000) - baseDate);
long maxBaseId = ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(ResourceSurrogateIdHelper.MaxDateTime);
Assert.Equal(ResourceSurrogateIdHelper.MaxDateTime.TruncateToMillisecond(), ResourceSurrogateIdHelper.ResourceSurrogateIdToLastUpdated(maxBaseId));
Assert.Equal(ResourceSurrogateIdHelper.MaxDateTime.TruncateToMillisecond(), ResourceSurrogateIdHelper.ResourceSurrogateIdToLastUpdated(maxBaseId + 79999));
}
public async Task GivenListOfResources_WhenBulkMergeToStoreTwice_ThenSecondMergeShouldFail()
{
List <SqlBulkCopyDataWrapper> resources = new List <SqlBulkCopyDataWrapper>();
long startSurrogateId = ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(DateTime.Now);
string resourceId = Guid.NewGuid().ToString();
SqlBulkCopyDataWrapper resource1 = CreateTestResource(resourceId, startSurrogateId);
SqlBulkCopyDataWrapper resource2 = CreateTestResource(resourceId, startSurrogateId + 1);
resources.Add(resource1);
resources.Add(resource2);
SqlBulkCopyDataWrapper[] result = (await _sqlServerFhirDataBulkOperation.BulkMergeResourceAsync(resources, CancellationToken.None)).ToArray();
int rCount = await GetResourceCountAsync("Resource", startSurrogateId, startSurrogateId + 2);
Assert.Single(result);
Assert.Equal(1, rCount);
}
public async Task GivenDuplicateResources_WhenBulkMergeToStore_ThenOnlyDistinctResourcesImported()
{
long startSurrogateId = ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(DateTime.Now);
int count = 100;
string resourceId = Guid.NewGuid().ToString();
List <SqlBulkCopyDataWrapper> resources = new List <SqlBulkCopyDataWrapper>();
for (int i = 0; i < count; ++i)
{
resources.Add(CreateTestResource(resourceId, startSurrogateId + i));
}
SqlBulkCopyDataWrapper[] result = (await _sqlServerFhirDataBulkOperation.BulkMergeResourceAsync(resources, CancellationToken.None)).ToArray();
int rCount = await GetResourceCountAsync("Resource", startSurrogateId, startSurrogateId + count);
Assert.Single(result);
Assert.Equal(1, rCount);
}
public async Task GivenBatchResources_WhenBulkCopy_ThenRecordsShouldBeAdded()
{
long startSurrogateId = ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(DateTime.Now);
int count = 1001;
short typeId = _fixture.SqlServerFhirModel.GetResourceTypeId("Patient");
await VerifyDataForBulkImport(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateDateTimeSearchParamsTable);
await VerifyDataForBulkImport(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateNumberSearchParamsTable);
await VerifyDataForBulkImport(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateQuantitySearchParamsTable);
await VerifyDataForBulkImport(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateReferenceSearchParamsTable);
await VerifyDataForBulkImport(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateReferenceTokenCompositeSearchParamsTable);
await VerifyDataForBulkImport(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateStringSearchParamsTable);
await VerifyDataForBulkImport(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateTokenDateTimeCompositeSearchParamsTable);
await VerifyDataForBulkImport(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateTokenNumberNumberCompositeSearchParamsTable);
await VerifyDataForBulkImport(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateTokenQuantityCompositeSearchParamsTable);
await VerifyDataForBulkImport(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateTokenSearchParamsTable);
await VerifyDataForBulkImport(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateTokenStringCompositeSearchParamsTable);
await VerifyDataForBulkImport(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateTokenTextSearchParamsTable);
await VerifyDataForBulkImport(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateTokenTokenCompositeSearchParamsTable);
await VerifyDataForBulkImport(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateUriSearchParamsTable);
await VerifyDataForBulkImport(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateCompartmentAssignmentTable);
await VerifyDataForBulkImport(_sqlServerFhirDataBulkOperation, startSurrogateId, count, typeId, TestBulkDataProvider.GenerateResourceWriteClaimTable);
}
private async Task <SearchResult> SearchImpl(SearchOptions searchOptions, bool historySearch, CancellationToken cancellationToken)
{
await _model.EnsureInitialized();
Expression searchExpression = searchOptions.Expression;
// AND in the continuation token
if (!string.IsNullOrWhiteSpace(searchOptions.ContinuationToken))
{
if (long.TryParse(searchOptions.ContinuationToken, NumberStyles.None, CultureInfo.InvariantCulture, out var token))
{
var tokenExpression = Expression.SearchParameter(SqlSearchParameters.ResourceSurrogateIdParameter, Expression.GreaterThan(SqlFieldName.ResourceSurrogateId, null, token));
searchExpression = searchExpression == null ? tokenExpression : (Expression)Expression.And(tokenExpression, searchExpression);
}
else
{
throw new BadRequestException(Resources.InvalidContinuationToken);
}
}
SqlRootExpression expression = (SqlRootExpression)searchExpression
?.AcceptVisitor(LastUpdatedToResourceSurrogateIdRewriter.Instance)
.AcceptVisitor(DateTimeEqualityRewriter.Instance)
.AcceptVisitor(FlatteningRewriter.Instance)
.AcceptVisitor(_sqlRootExpressionRewriter)
.AcceptVisitor(TableExpressionCombiner.Instance)
.AcceptVisitor(DenormalizedPredicateRewriter.Instance)
.AcceptVisitor(NormalizedPredicateReorderer.Instance)
.AcceptVisitor(_chainFlatteningRewriter)
.AcceptVisitor(DateTimeBoundedRangeRewriter.Instance)
.AcceptVisitor(_stringOverflowRewriter)
.AcceptVisitor(NumericRangeRewriter.Instance)
.AcceptVisitor(MissingSearchParamVisitor.Instance)
.AcceptVisitor(IncludeDenormalizedRewriter.Instance)
.AcceptVisitor(TopRewriter.Instance, searchOptions)
.AcceptVisitor(IncludeRewriter.Instance)
?? SqlRootExpression.WithDenormalizedExpressions();
using (var connection = new SqlConnection(_configuration.ConnectionString))
{
connection.Open();
using (SqlCommand sqlCommand = connection.CreateCommand())
{
var stringBuilder = new IndentedStringBuilder(new StringBuilder());
EnableTimeAndIoMessageLogging(stringBuilder, connection);
var queryGenerator = new SqlQueryGenerator(stringBuilder, new SqlQueryParameterManager(sqlCommand.Parameters), _model, historySearch);
expression.AcceptVisitor(queryGenerator, searchOptions);
sqlCommand.CommandText = stringBuilder.ToString();
LogSqlComand(sqlCommand);
using (var reader = await sqlCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken))
{
if (searchOptions.CountOnly)
{
await reader.ReadAsync(cancellationToken);
return(new SearchResult(reader.GetInt32(0), searchOptions.UnsupportedSearchParams));
}
var resources = new List <SearchResultEntry>(searchOptions.MaxItemCount);
long?newContinuationId = null;
bool moreResults = false;
int matchCount = 0;
while (await reader.ReadAsync(cancellationToken))
{
(short resourceTypeId, string resourceId, int version, bool isDeleted, long resourceSurrogateId, string requestMethod, bool isMatch, Stream rawResourceStream) = reader.ReadRow(
V1.Resource.ResourceTypeId,
V1.Resource.ResourceId,
V1.Resource.Version,
V1.Resource.IsDeleted,
V1.Resource.ResourceSurrogateId,
V1.Resource.RequestMethod,
_isMatch,
V1.Resource.RawResource);
// If we get to this point, we know there are more results so we need a continuation token
// Additionally, this resource shouldn't be included in the results
if (matchCount >= searchOptions.MaxItemCount && isMatch)
{
moreResults = true;
continue;
}
// See if this resource is a continuation token candidate and increase the count
if (isMatch)
{
newContinuationId = resourceSurrogateId;
matchCount++;
}
string rawResource;
using (rawResourceStream)
using (var gzipStream = new GZipStream(rawResourceStream, CompressionMode.Decompress))
using (var streamReader = new StreamReader(gzipStream, SqlServerFhirDataStore.ResourceEncoding))
{
rawResource = await streamReader.ReadToEndAsync();
}
resources.Add(new SearchResultEntry(
new ResourceWrapper(
resourceId,
version.ToString(CultureInfo.InvariantCulture),
_model.GetResourceTypeName(resourceTypeId),
new RawResource(rawResource, FhirResourceFormat.Json),
new ResourceRequest(requestMethod),
new DateTimeOffset(ResourceSurrogateIdHelper.ResourceSurrogateIdToLastUpdated(resourceSurrogateId), TimeSpan.Zero),
isDeleted,
null,
null,
null),
isMatch ? SearchEntryMode.Match : SearchEntryMode.Include));
}
// call NextResultAsync to get the info messages
await reader.NextResultAsync(cancellationToken);
IReadOnlyList <(string parameterName, string reason)> unsupportedSortingParameters;
if (searchOptions.Sort?.Count > 0)
{
// we don't currently support sort
unsupportedSortingParameters = searchOptions.UnsupportedSortingParams.Concat(searchOptions.Sort.Select(s => (s.searchParameterInfo.Name, Core.Resources.SortNotSupported))).ToList();
}
else
{
unsupportedSortingParameters = searchOptions.UnsupportedSortingParams;
}
return(new SearchResult(resources, searchOptions.UnsupportedSearchParams, unsupportedSortingParameters, moreResults ? newContinuationId.Value.ToString(CultureInfo.InvariantCulture) : null));
}
}
}
}
public void GivenADateTimeLargerThanTheLargestThatCanBeRepresentedAsASurrogateId_WhenTurnedIntoASurrogateId_Throws()
{
Assert.Throws <ArgumentOutOfRangeException>(() => ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(DateTime.MaxValue));
}
private async Task <SearchResult> SearchImpl(SearchOptions searchOptions, bool historySearch, CancellationToken cancellationToken)
{
Expression searchExpression = searchOptions.Expression;
// AND in the continuation token
if (!string.IsNullOrWhiteSpace(searchOptions.ContinuationToken) && !searchOptions.CountOnly)
{
var continuationToken = ContinuationToken.FromString(searchOptions.ContinuationToken);
if (continuationToken != null)
{
// in case it's a _lastUpdated sort optimization
if (string.IsNullOrEmpty(continuationToken.SortValue))
{
(SearchParameterInfo searchParamInfo, SortOrder sortOrder) = searchOptions.GetFirstSupportedSortParam();
Expression lastUpdatedExpression = sortOrder == SortOrder.Ascending ? Expression.GreaterThan(SqlFieldName.ResourceSurrogateId, null, continuationToken.ResourceSurrogateId)
: Expression.LessThan(SqlFieldName.ResourceSurrogateId, null, continuationToken.ResourceSurrogateId);
var tokenExpression = Expression.SearchParameter(SqlSearchParameters.ResourceSurrogateIdParameter, lastUpdatedExpression);
searchExpression = searchExpression == null ? tokenExpression : (Expression)Expression.And(tokenExpression, searchExpression);
}
}
else
{
throw new BadRequestException(Resources.InvalidContinuationToken);
}
}
if (searchOptions.CountOnly)
{
// if we're only returning a count, discard any _include parameters since included resources are not counted.
searchExpression = searchExpression?.AcceptVisitor(RemoveIncludesRewriter.Instance);
}
SqlRootExpression expression = (SqlRootExpression)searchExpression
?.AcceptVisitor(LastUpdatedToResourceSurrogateIdRewriter.Instance)
.AcceptVisitor(DateTimeEqualityRewriter.Instance)
.AcceptVisitor(FlatteningRewriter.Instance)
.AcceptVisitor(_sqlRootExpressionRewriter)
.AcceptVisitor(_sortRewriter, searchOptions)
.AcceptVisitor(DenormalizedPredicateRewriter.Instance)
.AcceptVisitor(NormalizedPredicateReorderer.Instance)
.AcceptVisitor(_chainFlatteningRewriter)
.AcceptVisitor(DateTimeBoundedRangeRewriter.Instance)
.AcceptVisitor(_stringOverflowRewriter)
.AcceptVisitor(NumericRangeRewriter.Instance)
.AcceptVisitor(MissingSearchParamVisitor.Instance)
.AcceptVisitor(IncludeDenormalizedRewriter.Instance)
.AcceptVisitor(TopRewriter.Instance, searchOptions)
.AcceptVisitor(IncludeRewriter.Instance)
?? SqlRootExpression.WithDenormalizedExpressions();
using (SqlConnectionWrapper sqlConnectionWrapper = _sqlConnectionWrapperFactory.ObtainSqlConnectionWrapper(true))
using (SqlCommandWrapper sqlCommandWrapper = sqlConnectionWrapper.CreateSqlCommand())
{
var stringBuilder = new IndentedStringBuilder(new StringBuilder());
EnableTimeAndIoMessageLogging(stringBuilder, sqlConnectionWrapper);
var queryGenerator = new SqlQueryGenerator(stringBuilder, new SqlQueryParameterManager(sqlCommandWrapper.Parameters), _model, historySearch, _schemaInformation);
expression.AcceptVisitor(queryGenerator, searchOptions);
sqlCommandWrapper.CommandText = stringBuilder.ToString();
LogSqlCommand(sqlCommandWrapper);
using (var reader = await sqlCommandWrapper.ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken))
{
if (searchOptions.CountOnly)
{
await reader.ReadAsync(cancellationToken);
return(new SearchResult(reader.GetInt32(0), searchOptions.UnsupportedSearchParams));
}
var resources = new List <SearchResultEntry>(searchOptions.MaxItemCount);
long?newContinuationId = null;
bool moreResults = false;
int matchCount = 0;
// Currently we support only date time sort type.
DateTime?sortValue = null;
while (await reader.ReadAsync(cancellationToken))
{
(short resourceTypeId, string resourceId, int version, bool isDeleted, long resourceSurrogateId, string requestMethod, bool isMatch, bool isPartialEntry, bool isRawResourceMetaSet, Stream rawResourceStream) = reader.ReadRow(
VLatest.Resource.ResourceTypeId,
VLatest.Resource.ResourceId,
VLatest.Resource.Version,
VLatest.Resource.IsDeleted,
VLatest.Resource.ResourceSurrogateId,
VLatest.Resource.RequestMethod,
_isMatch,
_isPartial,
VLatest.Resource.IsRawResourceMetaSet,
VLatest.Resource.RawResource);
// If we get to this point, we know there are more results so we need a continuation token
// Additionally, this resource shouldn't be included in the results
if (matchCount >= searchOptions.MaxItemCount && isMatch)
{
moreResults = true;
// At this point we are at the last row.
// if we have more columns, it means sort expressions were added.
if (reader.FieldCount > 10)
{
sortValue = reader.GetValue(SortValueColumnName) as DateTime?;
}
continue;
}
// See if this resource is a continuation token candidate and increase the count
if (isMatch)
{
newContinuationId = resourceSurrogateId;
matchCount++;
}
string rawResource;
using (rawResourceStream)
using (var gzipStream = new GZipStream(rawResourceStream, CompressionMode.Decompress))
using (var streamReader = new StreamReader(gzipStream, SqlServerFhirDataStore.ResourceEncoding))
{
rawResource = await streamReader.ReadToEndAsync();
}
// as long as at least one entry was marked as partial, this resultset
// should be marked as partial
_isResultPartial = _isResultPartial || isPartialEntry;
resources.Add(new SearchResultEntry(
new ResourceWrapper(
resourceId,
version.ToString(CultureInfo.InvariantCulture),
_model.GetResourceTypeName(resourceTypeId),
new RawResource(rawResource, FhirResourceFormat.Json, isMetaSet: isRawResourceMetaSet),
new ResourceRequest(requestMethod),
new DateTimeOffset(ResourceSurrogateIdHelper.ResourceSurrogateIdToLastUpdated(resourceSurrogateId), TimeSpan.Zero),
isDeleted,
null,
null,
null),
isMatch ? SearchEntryMode.Match : SearchEntryMode.Include));
}
// call NextResultAsync to get the info messages
await reader.NextResultAsync(cancellationToken);
IReadOnlyList <(string parameterName, string reason)> unsupportedSortingParameters;
if (searchOptions.Sort?.Count > 0)
{
unsupportedSortingParameters = searchOptions
.UnsupportedSortingParams
.Concat(searchOptions.Sort
.Where(x => !x.searchParameterInfo.IsSortSupported())
.Select(s => (s.searchParameterInfo.Name, Core.Resources.SortNotSupported))).ToList();
}
else
{
unsupportedSortingParameters = searchOptions.UnsupportedSortingParams;
}
// Continuation token prep
ContinuationToken continuationToken = null;
if (moreResults)
{
if (sortValue.HasValue)
{
continuationToken = new ContinuationToken(new object[]
{
sortValue.Value.ToString("o"),
newContinuationId ?? 0,
});
}
else
{
continuationToken = new ContinuationToken(new object[]
{
newContinuationId ?? 0,
});
}
}
return(new SearchResult(resources, searchOptions.UnsupportedSearchParams, unsupportedSortingParameters, continuationToken?.ToJson(), _isResultPartial));
}
}
}
private async Task <SearchResult> SearchImpl(SqlSearchOptions sqlSearchOptions, SqlSearchType searchType, string currentSearchParameterHash, CancellationToken cancellationToken)
{
Expression searchExpression = sqlSearchOptions.Expression;
// AND in the continuation token
if (!string.IsNullOrWhiteSpace(sqlSearchOptions.ContinuationToken) && !sqlSearchOptions.CountOnly)
{
var continuationToken = ContinuationToken.FromString(sqlSearchOptions.ContinuationToken);
if (continuationToken != null)
{
if (string.IsNullOrEmpty(continuationToken.SortValue))
{
// Check whether it's a _lastUpdated or (_type,_lastUpdated) sort optimization
bool optimize = true;
(SearchParameterInfo searchParamInfo, SortOrder sortOrder) = sqlSearchOptions.Sort.Count == 0 ? default : sqlSearchOptions.Sort[0];
if (sqlSearchOptions.Sort.Count > 0)
{
if (!(searchParamInfo.Name == SearchParameterNames.LastUpdated || searchParamInfo.Name == SearchParameterNames.ResourceType))
{
optimize = false;
}
}
FieldName fieldName;
object keyValue;
SearchParameterInfo parameter;
if (continuationToken.ResourceTypeId == null || _schemaInformation.Current < SchemaVersionConstants.PartitionedTables)
{
// backwards compat
parameter = SqlSearchParameters.ResourceSurrogateIdParameter;
fieldName = SqlFieldName.ResourceSurrogateId;
keyValue = continuationToken.ResourceSurrogateId;
}
else
{
parameter = SqlSearchParameters.PrimaryKeyParameter;
fieldName = SqlFieldName.PrimaryKey;
keyValue = new PrimaryKeyValue(continuationToken.ResourceTypeId.Value, continuationToken.ResourceSurrogateId);
}
Expression lastUpdatedExpression = null;
if (!optimize)
{
lastUpdatedExpression = Expression.GreaterThan(fieldName, null, keyValue);
}
else
{
if (sortOrder == SortOrder.Ascending)
{
lastUpdatedExpression = Expression.GreaterThan(fieldName, null, keyValue);
}
else
{
lastUpdatedExpression = Expression.LessThan(fieldName, null, keyValue);
}
}
var tokenExpression = Expression.SearchParameter(parameter, lastUpdatedExpression);
searchExpression = searchExpression == null ? tokenExpression : Expression.And(tokenExpression, searchExpression);
}
}
else
{
throw new BadRequestException(Resources.InvalidContinuationToken);
}
}
var originalSort = new List <(SearchParameterInfo, SortOrder)>(sqlSearchOptions.Sort);
var clonedSearchOptions = UpdateSort(sqlSearchOptions, searchExpression, searchType);
if (clonedSearchOptions.CountOnly)
{
// if we're only returning a count, discard any _include parameters since included resources are not counted.
searchExpression = searchExpression?.AcceptVisitor(RemoveIncludesRewriter.Instance);
}
SqlRootExpression expression = (SqlRootExpression)searchExpression
?.AcceptVisitor(LastUpdatedToResourceSurrogateIdRewriter.Instance)
.AcceptVisitor(DateTimeEqualityRewriter.Instance)
.AcceptVisitor(FlatteningRewriter.Instance)
.AcceptVisitor(UntypedReferenceRewriter.Instance)
.AcceptVisitor(_sqlRootExpressionRewriter)
.AcceptVisitor(_partitionEliminationRewriter)
.AcceptVisitor(_sortRewriter, clonedSearchOptions)
.AcceptVisitor(SearchParamTableExpressionReorderer.Instance)
.AcceptVisitor(MissingSearchParamVisitor.Instance)
.AcceptVisitor(NotExpressionRewriter.Instance)
.AcceptVisitor(_chainFlatteningRewriter)
.AcceptVisitor(ResourceColumnPredicatePushdownRewriter.Instance)
.AcceptVisitor(DateTimeBoundedRangeRewriter.Instance)
.AcceptVisitor(
(SqlExpressionRewriterWithInitialContext <object>)(_schemaInformation.Current >= SchemaVersionConstants.PartitionedTables
? StringOverflowRewriter.Instance
: LegacyStringOverflowRewriter.Instance))
.AcceptVisitor(NumericRangeRewriter.Instance)
.AcceptVisitor(IncludeMatchSeedRewriter.Instance)
.AcceptVisitor(TopRewriter.Instance, clonedSearchOptions)
.AcceptVisitor(IncludeRewriter.Instance)
?? SqlRootExpression.WithResourceTableExpressions();
using (SqlConnectionWrapper sqlConnectionWrapper = await _sqlConnectionWrapperFactory.ObtainSqlConnectionWrapperAsync(cancellationToken, true))
using (SqlCommandWrapper sqlCommandWrapper = sqlConnectionWrapper.CreateSqlCommand())
{
var stringBuilder = new IndentedStringBuilder(new StringBuilder());
EnableTimeAndIoMessageLogging(stringBuilder, sqlConnectionWrapper);
var queryGenerator = new SqlQueryGenerator(
stringBuilder,
new HashingSqlQueryParameterManager(new SqlQueryParameterManager(sqlCommandWrapper.Parameters)),
_model,
searchType,
_schemaInformation,
currentSearchParameterHash);
expression.AcceptVisitor(queryGenerator, clonedSearchOptions);
sqlCommandWrapper.CommandText = stringBuilder.ToString();
LogSqlCommand(sqlCommandWrapper);
using (var reader = await sqlCommandWrapper.ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken))
{
if (clonedSearchOptions.CountOnly)
{
await reader.ReadAsync(cancellationToken);
long count = reader.GetInt64(0);
if (count > int.MaxValue)
{
_requestContextAccessor.RequestContext.BundleIssues.Add(
new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Error,
OperationOutcomeConstants.IssueType.NotSupported,
string.Format(Core.Resources.SearchCountResultsExceedLimit, count, int.MaxValue)));
throw new InvalidSearchOperationException(string.Format(Core.Resources.SearchCountResultsExceedLimit, count, int.MaxValue));
}
var searchResult = new SearchResult((int)count, clonedSearchOptions.UnsupportedSearchParams);
// call NextResultAsync to get the info messages
await reader.NextResultAsync(cancellationToken);
return(searchResult);
}
var resources = new List <SearchResultEntry>(sqlSearchOptions.MaxItemCount);
short?newContinuationType = null;
long? newContinuationId = null;
bool moreResults = false;
int matchCount = 0;
string sortValue = null;
var isResultPartial = false;
int numberOfColumnsRead = 0;
while (await reader.ReadAsync(cancellationToken))
{
PopulateResourceTableColumnsToRead(
reader,
out short resourceTypeId,
out string resourceId,
out int version,
out bool isDeleted,
out long resourceSurrogateId,
out string requestMethod,
out bool isMatch,
out bool isPartialEntry,
out bool isRawResourceMetaSet,
out string searchParameterHash,
out Stream rawResourceStream);
numberOfColumnsRead = reader.FieldCount;
// If we get to this point, we know there are more results so we need a continuation token
// Additionally, this resource shouldn't be included in the results
if (matchCount >= clonedSearchOptions.MaxItemCount && isMatch)
{
moreResults = true;
continue;
}
string rawResource;
using (rawResourceStream)
{
rawResource = await _compressedRawResourceConverter.ReadCompressedRawResource(rawResourceStream);
}
// See if this resource is a continuation token candidate and increase the count
if (isMatch)
{
newContinuationType = resourceTypeId;
newContinuationId = resourceSurrogateId;
// For normal queries, we select _defaultNumberOfColumnsReadFromResult number of columns.
// If we have more, that means we have an extra column tracking sort value.
// Keep track of sort value if this is the last row.
if (matchCount == clonedSearchOptions.MaxItemCount - 1 && reader.FieldCount > _defaultNumberOfColumnsReadFromResult)
{
var tempSortValue = reader.GetValue(SortValueColumnName);
if ((tempSortValue as DateTime?) != null)
{
sortValue = (tempSortValue as DateTime?).Value.ToString("o");
}
else
{
sortValue = tempSortValue.ToString();
}
}
matchCount++;
}
// as long as at least one entry was marked as partial, this resultset
// should be marked as partial
isResultPartial = isResultPartial || isPartialEntry;
resources.Add(new SearchResultEntry(
new ResourceWrapper(
resourceId,
version.ToString(CultureInfo.InvariantCulture),
_model.GetResourceTypeName(resourceTypeId),
new RawResource(rawResource, FhirResourceFormat.Json, isMetaSet: isRawResourceMetaSet),
new ResourceRequest(requestMethod),
new DateTimeOffset(ResourceSurrogateIdHelper.ResourceSurrogateIdToLastUpdated(resourceSurrogateId), TimeSpan.Zero),
isDeleted,
null,
null,
null,
searchParameterHash),
isMatch ? SearchEntryMode.Match : SearchEntryMode.Include));
}
// call NextResultAsync to get the info messages
await reader.NextResultAsync(cancellationToken);
ContinuationToken continuationToken =
moreResults
? new ContinuationToken(
clonedSearchOptions.Sort.Select(s =>
s.searchParameterInfo.Name switch
{
SearchParameterNames.ResourceType => (object)newContinuationType,
SearchParameterNames.LastUpdated => newContinuationId,
_ => sortValue,
}).ToArray())
private async Task <SearchResult> SearchImpl(SearchOptions searchOptions, SqlSearchType searchType, string currentSearchParameterHash, CancellationToken cancellationToken)
{
Expression searchExpression = searchOptions.Expression;
// AND in the continuation token
if (!string.IsNullOrWhiteSpace(searchOptions.ContinuationToken) && !searchOptions.CountOnly)
{
var continuationToken = ContinuationToken.FromString(searchOptions.ContinuationToken);
if (continuationToken != null)
{
// in case it's a _lastUpdated sort optimization
if (string.IsNullOrEmpty(continuationToken.SortValue))
{
(SearchParameterInfo _, SortOrder sortOrder) = searchOptions.Sort.Count == 0 ? default : searchOptions.Sort[0];
Expression lastUpdatedExpression = sortOrder == SortOrder.Ascending
? Expression.GreaterThan(SqlFieldName.ResourceSurrogateId, null, continuationToken.ResourceSurrogateId)
: Expression.LessThan(SqlFieldName.ResourceSurrogateId, null, continuationToken.ResourceSurrogateId);
var tokenExpression = Expression.SearchParameter(SqlSearchParameters.ResourceSurrogateIdParameter, lastUpdatedExpression);
searchExpression = searchExpression == null ? tokenExpression : Expression.And(tokenExpression, searchExpression);
}
}
else
{
throw new BadRequestException(Resources.InvalidContinuationToken);
}
}
if (searchOptions.CountOnly)
{
// if we're only returning a count, discard any _include parameters since included resources are not counted.
searchExpression = searchExpression?.AcceptVisitor(RemoveIncludesRewriter.Instance);
}
SqlRootExpression expression = (SqlRootExpression)searchExpression
?.AcceptVisitor(LastUpdatedToResourceSurrogateIdRewriter.Instance)
.AcceptVisitor(DateTimeEqualityRewriter.Instance)
.AcceptVisitor(FlatteningRewriter.Instance)
.AcceptVisitor(UntypedReferenceRewriter.Instance)
.AcceptVisitor(_sqlRootExpressionRewriter)
.AcceptVisitor(_sortRewriter, searchOptions)
.AcceptVisitor(SearchParamTableExpressionReorderer.Instance)
.AcceptVisitor(MissingSearchParamVisitor.Instance)
.AcceptVisitor(NotExpressionRewriter.Instance)
.AcceptVisitor(_chainFlatteningRewriter)
.AcceptVisitor(ResourceColumnPredicatePushdownRewriter.Instance)
.AcceptVisitor(DateTimeBoundedRangeRewriter.Instance)
.AcceptVisitor(StringOverflowRewriter.Instance)
.AcceptVisitor(NumericRangeRewriter.Instance)
.AcceptVisitor(IncludeMatchSeedRewriter.Instance)
.AcceptVisitor(TopRewriter.Instance, searchOptions)
.AcceptVisitor(IncludeRewriter.Instance)
?? SqlRootExpression.WithResourceTableExpressions();
using (SqlConnectionWrapper sqlConnectionWrapper = await _sqlConnectionWrapperFactory.ObtainSqlConnectionWrapperAsync(cancellationToken, true))
using (SqlCommandWrapper sqlCommandWrapper = sqlConnectionWrapper.CreateSqlCommand())
{
var stringBuilder = new IndentedStringBuilder(new StringBuilder());
EnableTimeAndIoMessageLogging(stringBuilder, sqlConnectionWrapper);
var queryGenerator = new SqlQueryGenerator(
stringBuilder,
new SqlQueryParameterManager(sqlCommandWrapper.Parameters),
_model,
searchType,
_schemaInformation,
currentSearchParameterHash);
expression.AcceptVisitor(queryGenerator, searchOptions);
sqlCommandWrapper.CommandText = stringBuilder.ToString();
LogSqlCommand(sqlCommandWrapper);
using (var reader = await sqlCommandWrapper.ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken))
{
if (searchOptions.CountOnly)
{
await reader.ReadAsync(cancellationToken);
var searchResult = new SearchResult(reader.GetInt32(0), searchOptions.UnsupportedSearchParams);
// call NextResultAsync to get the info messages
await reader.NextResultAsync(cancellationToken);
return(searchResult);
}
var resources = new List <SearchResultEntry>(searchOptions.MaxItemCount);
long?newContinuationId = null;
bool moreResults = false;
int matchCount = 0;
// Currently we support only date time sort type.
DateTime?sortValue = null;
var isResultPartial = false;
while (await reader.ReadAsync(cancellationToken))
{
PopulateResourceTableColumnsToRead(
reader,
out short resourceTypeId,
out string resourceId,
out int version,
out bool isDeleted,
out long resourceSurrogateId,
out string requestMethod,
out bool isMatch,
out bool isPartialEntry,
out bool isRawResourceMetaSet,
out string searchParameterHash,
out Stream rawResourceStream);
// If we get to this point, we know there are more results so we need a continuation token
// Additionally, this resource shouldn't be included in the results
if (matchCount >= searchOptions.MaxItemCount && isMatch)
{
moreResults = true;
continue;
}
string rawResource;
using (rawResourceStream)
{
rawResource = await CompressedRawResourceConverter.ReadCompressedRawResource(rawResourceStream);
}
// See if this resource is a continuation token candidate and increase the count
if (isMatch)
{
newContinuationId = resourceSurrogateId;
// Keep track of sort value if this is the last row.
// if we have more than 10 columns, it means sort expressions were added.
if (matchCount == searchOptions.MaxItemCount - 1 && reader.FieldCount > _resourceTableColumnCount + 1)
{
sortValue = reader.GetValue(SortValueColumnName) as DateTime?;
}
matchCount++;
}
// as long as at least one entry was marked as partial, this resultset
// should be marked as partial
isResultPartial = isResultPartial || isPartialEntry;
resources.Add(new SearchResultEntry(
new ResourceWrapper(
resourceId,
version.ToString(CultureInfo.InvariantCulture),
_model.GetResourceTypeName(resourceTypeId),
new RawResource(rawResource, FhirResourceFormat.Json, isMetaSet: isRawResourceMetaSet),
new ResourceRequest(requestMethod),
new DateTimeOffset(ResourceSurrogateIdHelper.ResourceSurrogateIdToLastUpdated(resourceSurrogateId), TimeSpan.Zero),
isDeleted,
null,
null,
null,
searchParameterHash),
isMatch ? SearchEntryMode.Match : SearchEntryMode.Include));
}
// call NextResultAsync to get the info messages
await reader.NextResultAsync(cancellationToken);
// Continuation token prep
ContinuationToken continuationToken = null;
if (moreResults)
{
if (sortValue.HasValue)
{
continuationToken = new ContinuationToken(new object[]
{
sortValue.Value.ToString("o"),
newContinuationId ?? 0,
});
}
else
{
continuationToken = new ContinuationToken(new object[]
{
newContinuationId ?? 0,
});
}
}
if (isResultPartial)
{
_requestContextAccessor.FhirRequestContext.BundleIssues.Add(
new OperationOutcomeIssue(
OperationOutcomeConstants.IssueSeverity.Warning,
OperationOutcomeConstants.IssueType.Incomplete,
Core.Resources.TruncatedIncludeMessage));
}
return(new SearchResult(resources, continuationToken?.ToJson(), searchOptions.Sort, searchOptions.UnsupportedSearchParams));
}
}
}
/// <summary>
/// Get current surrogateId from datetime
/// </summary>
/// <returns>Current surrogated id.</returns>
public long GetCurrentSequenceId()
{
return(ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(DateTime.UtcNow));
}
