private async Task ProcessQueryAsync(ReindexJobQueryStatus query, CancellationToken cancellationToken)
{
try
{
query.Status = OperationStatus.Running;
query.LastModified = DateTimeOffset.UtcNow;
// Query first batch of resources
var results = await ExecuteReindexQueryAsync(query, false, cancellationToken);
// if continuation token then update next query
if (!string.IsNullOrEmpty(results.ContinuationToken))
{
var nextQuery = new ReindexJobQueryStatus(results.ContinuationToken);
nextQuery.LastModified = DateTimeOffset.UtcNow;
nextQuery.Status = OperationStatus.Queued;
_reindexJobRecord.QueryList.Add(nextQuery);
}
await UpdateJobAsync(cancellationToken);
// TODO: Release lock on job document so another thread may pick up the next query.
await _reindexUtilities.ProcessSearchResultsAsync(results, _reindexJobRecord.Hash, cancellationToken);
// TODO: reaquire document lock and update _etag
query.Status = OperationStatus.Completed;
await UpdateJobAsync(cancellationToken);
await CheckJobCompletionStatus(cancellationToken);
}
catch (Exception ex)
{
query.Error = ex.Message;
query.FailureCount++;
_logger.LogError(ex, $"Encountered an unhandled exception. The query failure count increased to {_reindexJobRecord.FailureCount}.");
if (query.FailureCount >= _reindexJobConfiguration.ConsecutiveFailuresThreshold)
{
query.Status = OperationStatus.Failed;
}
else
{
query.Status = OperationStatus.Queued;
}
await UpdateJobAsync(cancellationToken);
}
}
private async Task <ReindexJobQueryStatus> ProcessQueryAsync(ReindexJobQueryStatus query, SemaphoreSlim jobSemaphore, CancellationToken cancellationToken)
{
try
{
SearchResult results;
await jobSemaphore.WaitAsync();
try
{
query.Status = OperationStatus.Running;
query.LastModified = DateTimeOffset.UtcNow;
// Query first batch of resources
results = await ExecuteReindexQueryAsync(query, countOnly : false, cancellationToken);
// if continuation token then update next query
if (!string.IsNullOrEmpty(results?.ContinuationToken))
{
var encodedContinuationToken = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(results.ContinuationToken));
var nextQuery = new ReindexJobQueryStatus(query.ResourceType, encodedContinuationToken)
{
LastModified = Clock.UtcNow,
Status = OperationStatus.Queued,
};
_reindexJobRecord.QueryList.Add(nextQuery);
}
await UpdateJobAsync(cancellationToken);
}
finally
{
jobSemaphore.Release();
}
_logger.LogInformation($"Reindex job current thread: {Thread.CurrentThread.ManagedThreadId}");
await _reindexUtilities.ProcessSearchResultsAsync(results, _reindexJobRecord.ResourceTypeSearchParameterHashMap, cancellationToken);
if (!cancellationToken.IsCancellationRequested)
{
await jobSemaphore.WaitAsync();
try
{
_reindexJobRecord.Progress += results.Results.Count();
query.Status = OperationStatus.Completed;
await UpdateJobAsync(cancellationToken);
}
finally
{
jobSemaphore.Release();
}
}
return(query);
}
catch (Exception ex)
{
await jobSemaphore.WaitAsync();
try
{
query.Error = ex.Message;
query.FailureCount++;
_logger.LogError(ex, $"Encountered an unhandled exception. The query failure count increased to {_reindexJobRecord.FailureCount}.");
if (query.FailureCount >= _reindexJobConfiguration.ConsecutiveFailuresThreshold)
{
query.Status = OperationStatus.Failed;
}
else
{
query.Status = OperationStatus.Queued;
}
await UpdateJobAsync(cancellationToken);
}
finally
{
jobSemaphore.Release();
}
return(query);
}
}
private async Task <ReindexJobQueryStatus> ProcessQueryAsync(ReindexJobQueryStatus query, SemaphoreSlim jobSemaphore, CancellationToken cancellationToken)
{
try
{
SearchResult results;
await jobSemaphore.WaitAsync();
try
{
query.Status = OperationStatus.Running;
query.LastModified = DateTimeOffset.UtcNow;
// Query first batch of resources
results = await ExecuteReindexQueryAsync(query, countOnly : false, cancellationToken);
// if continuation token then update next query
if (!string.IsNullOrEmpty(results?.ContinuationToken))
{
var encodedContinuationToken = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(results.ContinuationToken));
var nextQuery = new ReindexJobQueryStatus(query.ResourceType, encodedContinuationToken)
{
LastModified = Clock.UtcNow,
Status = OperationStatus.Queued,
};
_reindexJobRecord.QueryList.TryAdd(nextQuery, 1);
}
await UpdateJobAsync(cancellationToken);
_throttleController.UpdateDatastoreUsage();
}
finally
{
jobSemaphore.Release();
}
_logger.LogInformation($"Reindex job current thread: {Thread.CurrentThread.ManagedThreadId}");
await _reindexUtilities.ProcessSearchResultsAsync(results, _reindexJobRecord.ResourceTypeSearchParameterHashMap, cancellationToken);
_throttleController.UpdateDatastoreUsage();
if (!cancellationToken.IsCancellationRequested)
{
await jobSemaphore.WaitAsync();
try
{
_logger.LogInformation("Reindex job updating progress, current result count: {0}", results.Results.Count());
_reindexJobRecord.Progress += results.Results.Count();
query.Status = OperationStatus.Completed;
// Remove oldest completed queryStatus object if count > 10
// to ensure reindex job document doesn't grow too large
if (_reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Completed).Count() > 10)
{
var queryStatusToRemove = _reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Completed).OrderBy(q => q.LastModified).FirstOrDefault();
_reindexJobRecord.QueryList.TryRemove(queryStatusToRemove, out var removedByte);
}
await UpdateJobAsync(cancellationToken);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Reindex error occurred recording progress.");
throw;
}
finally
{
jobSemaphore.Release();
}
}
return(query);
}
catch (Exception ex)
{
await jobSemaphore.WaitAsync();
try
{
query.Error = ex.Message;
query.FailureCount++;
_logger.LogError(ex, "Encountered an unhandled exception. The query failure count increased to {failureCount}.", _reindexJobRecord.FailureCount);
if (query.FailureCount >= _reindexJobConfiguration.ConsecutiveFailuresThreshold)
{
query.Status = OperationStatus.Failed;
}
else
{
query.Status = OperationStatus.Queued;
}
await UpdateJobAsync(cancellationToken);
}
finally
{
jobSemaphore.Release();
}
return(query);
}
}
private async Task <ReindexJobQueryStatus> ProcessQueryAsync(ReindexJobQueryStatus query, CancellationToken cancellationToken)
{
try
{
SearchResult results;
await _jobSemaphore.WaitAsync(cancellationToken);
try
{
// Query first batch of resources
results = await ExecuteReindexQueryAsync(query, countOnly : false, cancellationToken);
// If continuation token then update next query but only if parent query haven't been in pipeline.
// For cases like retry or stale query we don't want to start another chain.
if (!string.IsNullOrEmpty(results?.ContinuationToken) && !query.CreatedChild)
{
var encodedContinuationToken = ContinuationTokenConverter.Encode(results.ContinuationToken);
var nextQuery = new ReindexJobQueryStatus(query.ResourceType, encodedContinuationToken)
{
LastModified = Clock.UtcNow,
Status = OperationStatus.Queued,
};
_reindexJobRecord.QueryList.TryAdd(nextQuery, 1);
query.CreatedChild = true;
}
await UpdateJobAsync();
_throttleController.UpdateDatastoreUsage();
}
finally
{
_jobSemaphore.Release();
}
_logger.LogInformation($"Reindex job current thread: {Thread.CurrentThread.ManagedThreadId}");
await _reindexUtilities.ProcessSearchResultsAsync(results, _reindexJobRecord.ResourceTypeSearchParameterHashMap, cancellationToken);
_throttleController.UpdateDatastoreUsage();
if (!_cancellationToken.IsCancellationRequested)
{
await _jobSemaphore.WaitAsync(cancellationToken);
try
{
_logger.LogInformation("Reindex job updating progress, current result count: {0}", results.Results.Count());
_reindexJobRecord.Progress += results.Results.Count();
query.Status = OperationStatus.Completed;
// Remove oldest completed queryStatus object if count > 10
// to ensure reindex job document doesn't grow too large
if (_reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Completed).Count() > 10)
{
var queryStatusToRemove = _reindexJobRecord.QueryList.Keys.Where(q => q.Status == OperationStatus.Completed).OrderBy(q => q.LastModified).FirstOrDefault();
_reindexJobRecord.QueryList.TryRemove(queryStatusToRemove, out var removedByte);
}
await UpdateJobAsync();
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Reindex error occurred recording progress.");
throw;
}
finally
{
_jobSemaphore.Release();
}
}
return(query);
}
catch (FhirException ex)
{
return(await HandleQueryException(query, ex, true, cancellationToken));
}
catch (Exception ex)
{
return(await HandleQueryException(query, ex, false, cancellationToken));
}
}
