public void GivenShortBase64WhenDecoding_ThenCorrectValueIsReturned()
{
var data = "YWJj";
var decoded = ContinuationTokenConverter.Decode(data);
Assert.Equal("abc", decoded);
}
public void GivenAnInvalidString_WhenDecoding_ThenAnErrorIsThrown()
{
var data = Guid.NewGuid().ToString();
var encodedPrevious = Convert.ToBase64String(Encoding.UTF8.GetBytes(data)).Insert(5, "aaaafffff");
Assert.Throws <BadRequestException>(() => ContinuationTokenConverter.Decode(encodedPrevious));
}
public void GivenAString_WhenEcodingAndDecoding_ThenOriginalStringIsPreserved()
{
var data = Guid.NewGuid().ToString();
var encoded = ContinuationTokenConverter.Encode(data);
var decoded = ContinuationTokenConverter.Decode(encoded);
Assert.Equal(data, decoded);
}
public void GivenAnOldStringInBase64_WhenDecoding_ThenOriginalStringIsPreserved()
{
var data = Guid.NewGuid().ToString();
var encodedPrevious = Convert.ToBase64String(Encoding.UTF8.GetBytes(data));
var decoded = ContinuationTokenConverter.Decode(encodedPrevious);
Assert.Equal(data, decoded);
}
public void GivenASearchResultWithContinuationToken_WhenCreateSearchBundle_ThenCorrectBundleShouldBeReturned()
{
string encodedContinuationToken = ContinuationTokenConverter.Encode(_continuationToken);
_urlResolver.ResolveRouteUrl(_unsupportedSearchParameters, null, encodedContinuationToken, true).Returns(_nextUrl);
_urlResolver.ResolveRouteUrl(_unsupportedSearchParameters).Returns(_selfUrl);
var searchResult = new SearchResult(new SearchResultEntry[0], _continuationToken, null, _unsupportedSearchParameters);
ResourceElement actual = _bundleFactory.CreateSearchBundle(searchResult);
Assert.Equal(_nextUrl.OriginalString, actual.Scalar <string>("Bundle.link.where(relation='next').url"));
}
private async Task <List <ArtifactSummary> > GetSummaries()
{
var result = new Dictionary <string, ArtifactSummary>();
using (IScoped <ISearchService> searchService = _searchServiceFactory())
{
foreach (var type in _supportedTypes)
{
string ct = null;
{
do
{
var queryParameters = new List <Tuple <string, string> >();
if (ct != null)
{
ct = ContinuationTokenConverter.Encode(ct);
queryParameters.Add(new Tuple <string, string>(KnownQueryParameterNames.ContinuationToken, ct));
}
var searchResult = await searchService.Value.SearchAsync(type, queryParameters, CancellationToken.None);
foreach (var searchItem in searchResult.Results)
{
using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(searchItem.Resource.RawResource.Data)))
{
using var navStream = new JsonNavigatorStream(memoryStream);
Action <ArtifactSummaryPropertyBag> setOrigin =
(properties) =>
{
properties[ArtifactSummaryProperties.OriginKey] = searchItem.Resource.RawResource.Data;
};
var artifacts = ArtifactSummaryGenerator.Default.Generate(navStream, setOrigin);
foreach (var artifact in artifacts)
{
result[artifact.ResourceUri] = artifact;
}
}
}
ct = searchResult.ContinuationToken;
}while (ct != null);
}
}
return(result.Values.ToList());
}
}
private async Task ProcessProgressChange(
ExportJobConfiguration exportJobConfiguration,
ExportJobProgress progress,
List <Tuple <string, string> > queryParametersList,
string continuationToken,
bool forceCommit,
CancellationToken cancellationToken)
{
// Update the continuation token in local cache and queryParams.
// We will add or udpate the continuation token in the query parameters list.
progress.UpdateContinuationToken(ContinuationTokenConverter.Encode(continuationToken));
bool replacedContinuationToken = false;
for (int index = 0; index < queryParametersList.Count; index++)
{
if (queryParametersList[index].Item1 == KnownQueryParameterNames.ContinuationToken)
{
queryParametersList[index] = Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken);
replacedContinuationToken = true;
}
}
if (!replacedContinuationToken)
{
queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, progress.ContinuationToken));
}
if (progress.Page % _exportJobRecord.NumberOfPagesPerCommit == 0 || forceCommit)
{
// Commit the changes.
await _exportDestinationClient.CommitAsync(exportJobConfiguration, cancellationToken);
// Update the job record.
await UpdateJobRecordAsync(cancellationToken);
}
}
private async Task <ReindexJobQueryStatus> ProcessQueryAsync(ReindexJobQueryStatus query, CancellationToken cancellationToken)
{
try
{
SearchResult results;
await _jobSemaphore.WaitAsync(cancellationToken);
try
{
// Query first batch of resources
results = await ExecuteReindexQueryAsync(query, countOnly : false, cancellationToken);
// If continuation token then update next query but only if parent query haven't been in pipeline.
// For cases like retry or stale query we don't want to start another chain.
if (!string.IsNullOrEmpty(results?.ContinuationToken) && !query.CreatedChild)
{
var encodedContinuationToken = ContinuationTokenConverter.Encode(results.ContinuationToken);
var nextQuery = new ReindexJobQueryStatus(query.ResourceType, encodedContinuationToken)
{
LastModified = Clock.UtcNow,
Status = OperationStatus.Queued,
};
_reindexJobRecord.QueryList.TryAdd(nextQuery, 1);
query.CreatedChild = true;
}
await UpdateJobAsync();
_throttleController.UpdateDatastoreUsage();
}
finally
{
_jobSemaphore.Release();
}
_logger.LogInformation($"Reindex job current thread: {Thread.CurrentThread.ManagedThreadId}");
await _reindexUtilities.ProcessSearchResultsAsync(results, _reindexJobRecord.ResourceTypeSearchParameterHashMap, cancellationToken);
_throttleController.UpdateDatastoreUsage();
if (!_cancellationToken.IsCancellationRequested)
{
await _jobSemaphore.WaitAsync(cancellationToken);
try
{
_logger.LogInformation("Reindex job updating progress, current result count: {0}", results.Results.Count());
_reindexJobRecord.Progress += results.Results.Count();
query.Status = OperationStatus.Completed;
// Remove oldest completed queryStatus object if count > 10
// to ensure reindex job document doesn't grow too large
if (_reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Completed).Count() > 10)
{
var queryStatusToRemove = _reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Completed).OrderBy(q => q.LastModified).FirstOrDefault();
_reindexJobRecord.QueryList.TryRemove(queryStatusToRemove, out var removedByte);
}
await UpdateJobAsync();
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Reindex error occurred recording progress.");
throw;
}
finally
{
_jobSemaphore.Release();
}
}
return(query);
}
catch (FhirException ex)
{
return(await HandleQueryException(query, ex, true, cancellationToken));
}
catch (Exception ex)
{
return(await HandleQueryException(query, ex, false, cancellationToken));
}
}
