public async Task GivenThereIsARunningReindexJob_WhenSimultaneousUpdateCallsOccur_ThenJobConflictExceptionShouldBeThrown()
{
ReindexJobWrapper runningJobWrapper = await CreateRunningReindexJob();
var completionSource = new TaskCompletionSource <bool>();
Task <ReindexJobWrapper>[] tasks = new[]
{
WaitAndUpdateReindexJobAsync(runningJobWrapper),
WaitAndUpdateReindexJobAsync(runningJobWrapper),
WaitAndUpdateReindexJobAsync(runningJobWrapper),
};
completionSource.SetResult(true);
await Assert.ThrowsAsync <JobConflictException>(() => Task.WhenAll(tasks));
async Task <ReindexJobWrapper> WaitAndUpdateReindexJobAsync(ReindexJobWrapper jobWrapper)
{
await completionSource.Task;
jobWrapper.JobRecord.Status = OperationStatus.Completed;
return(await _operationDataStore.UpdateReindexJobAsync(jobWrapper.JobRecord, jobWrapper.ETag, CancellationToken.None));
}
}
public async Task <ReindexJobWrapper> GetReindexJobByIdAsync(string jobId, CancellationToken cancellationToken)
{
EnsureArg.IsNotNullOrWhiteSpace(jobId, nameof(jobId));
try
{
DocumentResponse <CosmosReindexJobRecordWrapper> cosmosReindexJobRecord = await _documentClientScope.Value.ReadDocumentAsync <CosmosReindexJobRecordWrapper>(
UriFactory.CreateDocumentUri(DatabaseId, CollectionId, jobId),
new RequestOptions { PartitionKey = new PartitionKey(CosmosDbReindexConstants.ReindexJobPartitionKey) },
cancellationToken);
var outcome = new ReindexJobWrapper(cosmosReindexJobRecord.Document.JobRecord, WeakETag.FromVersionId(cosmosReindexJobRecord.Document.ETag));
return(outcome);
}
catch (DocumentClientException dce)
{
if (dce.StatusCode == HttpStatusCode.TooManyRequests)
{
throw new RequestRateExceededException(dce.RetryAfter);
}
else if (dce.StatusCode == HttpStatusCode.NotFound)
{
throw new JobNotFoundException(string.Format(Core.Resources.JobNotFound, jobId));
}
_logger.LogError(dce, $"Failed to get reindex job by id: {jobId}.");
throw;
}
}
public async Task <CancelReindexResponse> Handle(CancelReindexRequest request, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(request, nameof(request));
if (await _authorizationService.CheckAccess(DataActions.Reindex, cancellationToken) != DataActions.Reindex)
{
throw new UnauthorizedFhirActionException();
}
return(await _retryPolicy.ExecuteAsync(async() =>
{
ReindexJobWrapper outcome = await _fhirOperationDataStore.GetReindexJobByIdAsync(request.JobId, cancellationToken);
// If the job is already completed for any reason, return conflict status.
if (outcome.JobRecord.Status.IsFinished())
{
throw new RequestNotValidException(
string.Format(Resources.ReindexJobInCompletedState, outcome.JobRecord.Id, outcome.JobRecord.Status));
}
// Try to cancel the job.
outcome.JobRecord.Status = OperationStatus.Canceled;
outcome.JobRecord.CanceledTime = Clock.UtcNow;
await _fhirOperationDataStore.UpdateReindexJobAsync(outcome.JobRecord, outcome.ETag, cancellationToken);
return new CancelReindexResponse(HttpStatusCode.Conflict, outcome);
}));
}
public async Task <ReindexJobWrapper> GetReindexJobByIdAsync(string jobId, CancellationToken cancellationToken)
{
EnsureArg.IsNotNullOrWhiteSpace(jobId, nameof(jobId));
try
{
var cosmosReindexJobRecord = await _containerScope.Value.ReadItemAsync <CosmosReindexJobRecordWrapper>(
jobId,
new PartitionKey(CosmosDbReindexConstants.ReindexJobPartitionKey),
cancellationToken : cancellationToken);
var outcome = new ReindexJobWrapper(
cosmosReindexJobRecord.Resource.JobRecord,
WeakETag.FromVersionId(cosmosReindexJobRecord.Resource.ETag));
return(outcome);
}
catch (CosmosException dce)
{
if (dce.IsRequestRateExceeded())
{
throw;
}
else if (dce.StatusCode == HttpStatusCode.NotFound)
{
throw new JobNotFoundException(string.Format(Core.Resources.JobNotFound, jobId));
}
_logger.LogError(dce, $"Failed to get reindex job by id: {jobId}.");
throw;
}
}
public static ResourceElement ToParametersResourceElement(this ReindexJobWrapper record)
{
EnsureArg.IsNotNull(record, nameof(record));
var parametersResource = new Parameters();
parametersResource.VersionId = record.ETag.VersionId;
var job = record.JobRecord;
parametersResource.Id = job.Id;
parametersResource.Parameter = new List <Parameters.ParameterComponent>();
parametersResource.Parameter.Add(new Parameters.ParameterComponent()
{
Name = JobRecordProperties.Id, Value = new FhirString(job.Id)
});
if (job.StartTime.HasValue)
{
parametersResource.Parameter.Add(new Parameters.ParameterComponent()
{
Name = JobRecordProperties.StartTime, Value = new FhirDateTime(job.StartTime.Value)
});
}
if (job.EndTime.HasValue)
{
parametersResource.Parameter.Add(new Parameters.ParameterComponent()
{
Name = JobRecordProperties.EndTime, Value = new FhirDateTime(job.EndTime.Value)
});
}
parametersResource.Parameter.Add(new Parameters.ParameterComponent()
{
Name = JobRecordProperties.QueuedTime, Value = new FhirDateTime(job.QueuedTime)
});
parametersResource.Parameter.Add(new Parameters.ParameterComponent()
{
Name = JobRecordProperties.Progress, Value = new FhirDecimal(job.PercentComplete)
});
parametersResource.Parameter.Add(new Parameters.ParameterComponent()
{
Name = JobRecordProperties.Status, Value = new FhirString(job.Status.ToString())
});
parametersResource.Parameter.Add(new Parameters.ParameterComponent()
{
Name = JobRecordProperties.MaximumConcurrency, Value = new FhirDecimal(job.MaximumConcurrency)
});
parametersResource.Parameter.Add(new Parameters.ParameterComponent()
{
Name = JobRecordProperties.Resources, Value = new FhirString(job.ResourceList)
});
parametersResource.Parameter.Add(new Parameters.ParameterComponent()
{
Name = JobRecordProperties.SearchParams, Value = new FhirString(job.SearchParamList)
});
return(parametersResource.ToResourceElement());
}
public async Task GivenAMatchingReindexJob_WhenGettingById_ThenTheMatchingReindexJobShouldBeReturned()
{
ReindexJobRecord jobRecord = await InsertNewReindexJobRecordAsync();
ReindexJobWrapper jobWrapper = await _operationDataStore.GetReindexJobByIdAsync(jobRecord.Id, CancellationToken.None);
Assert.Equal(jobRecord.Id, jobWrapper?.JobRecord?.Id);
}
private static GetReindexResponse GetReindexJobResponse()
{
var jobRecord = new ReindexJobRecord(_searchParameterHashMap, new List <string>(), 5);
var jobWrapper = new ReindexJobWrapper(
jobRecord,
WeakETag.FromVersionId("33a64df551425fcc55e4d42a148795d9f25f89d4"));
return(new GetReindexResponse(System.Net.HttpStatusCode.OK, jobWrapper));
}
private static GetReindexResponse GetReindexJobResponse()
{
var jobRecord = new ReindexJobRecord("hash", 5, "patient");
var jobWrapper = new ReindexJobWrapper(
jobRecord,
WeakETag.FromVersionId("33a64df551425fcc55e4d42a148795d9f25f89d4"));
return(new GetReindexResponse(System.Net.HttpStatusCode.OK, jobWrapper));
}
private static CreateReindexResponse GetCreateReindexResponse()
{
var jobRecord = new ReindexJobRecord("hash", 5, "patient");
var jobWrapper = new ReindexJobWrapper(
jobRecord,
WeakETag.FromVersionId("33a64df551425fcc55e4d42a148795d9f25f89d4"));
return(new CreateReindexResponse(jobWrapper));
}
private static CreateReindexResponse GetCreateReindexResponse()
{
var jobRecord = new ReindexJobRecord(_searchParameterHashMap, 5);
var jobWrapper = new ReindexJobWrapper(
jobRecord,
WeakETag.FromVersionId("33a64df551425fcc55e4d42a148795d9f25f89d4"));
return(new CreateReindexResponse(jobWrapper));
}
private async Task <ReindexJobRecord> InsertNewReindexJobRecordAsync(Action <ReindexJobRecord> jobRecordCustomizer = null)
{
var jobRecord = new ReindexJobRecord("searchParamHash", maxiumumConcurrency: 1, scope: "all");
jobRecordCustomizer?.Invoke(jobRecord);
ReindexJobWrapper result = await _operationDataStore.CreateReindexJobAsync(jobRecord, CancellationToken.None);
return(result.JobRecord);
}
public async Task GivenANonexistentReindexJob_WhenUpdatingTheReindexJob_ThenJobNotFoundExceptionShouldBeThrown()
{
ReindexJobWrapper jobWrapper = await CreateRunningReindexJob();
ReindexJobRecord job = jobWrapper.JobRecord;
WeakETag jobVersion = jobWrapper.ETag;
await _testHelper.DeleteReindexJobRecordAsync(job.Id);
await Assert.ThrowsAsync <JobNotFoundException>(() => _operationDataStore.UpdateReindexJobAsync(job, jobVersion, CancellationToken.None));
}
public async Task GivenThereIsNoRunningJob_WhenExecuted_ThenATaskShouldBeCreated()
{
ReindexJobWrapper job = CreateReindexJobWrapper();
SetupOperationDataStore(job);
_cancellationTokenSource.CancelAfter(DefaultJobPollingFrequency);
await _reindexJobWorker.ExecuteAsync(_cancellationToken);
_reindexJobTaskFactory().Received(1);
}
private async Task <GetReindexResponse> GetSingleReindexJobAsync(string jobId, CancellationToken cancellationToken)
{
try
{
ReindexJobWrapper reindexJob = await _fhirOperationDataStore.GetReindexJobByIdAsync(jobId, cancellationToken);
return(new GetReindexResponse(HttpStatusCode.OK, reindexJob));
}
catch (Exception ex)
{
throw new OperationFailedException($"Unable to read reindex job with id {jobId}, error: {ex.Message}", HttpStatusCode.BadRequest);
}
}
public async Task GivenAGetRequest_WhenTooManyRequestsThrown_ThenTooManyRequestsThrown()
{
var request = new GetReindexRequest("id");
var jobRecord = new ReindexJobRecord("hash", 1, null);
var jobWrapper = new ReindexJobWrapper(jobRecord, WeakETag.FromVersionId("id"));
_fhirOperationDataStore.GetReindexJobByIdAsync("id", CancellationToken.None).Throws(new RequestRateExceededException(TimeSpan.FromMilliseconds(100)));
var handler = new GetReindexRequestHandler(_fhirOperationDataStore, DisabledFhirAuthorizationService.Instance);
await Assert.ThrowsAsync <RequestRateExceededException>(() => handler.Handle(request, CancellationToken.None));
}
public async Task GivenThereIsRunningReindexJobThatExpired_WhenAcquiringReindexJobs_ThenTheExpiredReindexJobShouldBeReturned()
{
ReindexJobWrapper jobWrapper = await CreateRunningReindexJob();
await Task.Delay(1200);
IReadOnlyCollection <ReindexJobWrapper> expiredJobs = await AcquireReindexJobsAsync(jobHeartbeatTimeoutThreshold : TimeSpan.FromSeconds(1));
Assert.NotNull(expiredJobs);
Assert.Collection(
expiredJobs,
expiredJobWrapper => ValidateReindexJobRecord(jobWrapper.JobRecord, expiredJobWrapper.JobRecord));
}
public async Task GivenAGetRequest_WhenIdNotFound_ThenJobNotFoundExceptionThrown()
{
var request = new GetReindexRequest("id");
var jobRecord = new ReindexJobRecord("hash", 1, null);
var jobWrapper = new ReindexJobWrapper(jobRecord, WeakETag.FromVersionId("id"));
_fhirOperationDataStore.GetReindexJobByIdAsync("id", Arg.Any <CancellationToken>()).Throws(new JobNotFoundException("not found"));
var handler = new GetReindexRequestHandler(_fhirOperationDataStore, DisabledFhirAuthorizationService.Instance);
await Assert.ThrowsAsync <JobNotFoundException>(() => handler.Handle(request, CancellationToken.None));
}
private async Task <ReindexJobRecord> InsertNewReindexJobRecordAsync(Action <ReindexJobRecord> jobRecordCustomizer = null)
{
Dictionary <string, string> searchParamHashMap = new Dictionary <string, string>();
searchParamHashMap.Add("Patient", "searchParamHash");
var jobRecord = new ReindexJobRecord(searchParamHashMap, maxiumumConcurrency: 1, scope: "all");
jobRecordCustomizer?.Invoke(jobRecord);
ReindexJobWrapper result = await _operationDataStore.CreateReindexJobAsync(jobRecord, CancellationToken.None);
return(result.JobRecord);
}
public async Task GivenTheNumberOfRunningJobEqualsThreshold_WhenExecuted_ThenATaskShouldNotBeCreated()
{
ReindexJobWrapper job = CreateReindexJobWrapper();
SetupOperationDataStore(job);
_task.ExecuteAsync(job.JobRecord, job.ETag, _cancellationToken).Returns(Task.Run(async() => { await Task.Delay(1000); }));
_cancellationTokenSource.CancelAfter(DefaultJobPollingFrequency * 2);
await _reindexJobWorker.ExecuteAsync(_cancellationToken);
_reindexJobTaskFactory.Received(1);
}
public async Task GivenARunningReindexJob_WhenUpdatingTheReindexJob_ThenTheJobShouldBeUpdated()
{
ReindexJobWrapper jobWrapper = await CreateRunningReindexJob();
ReindexJobRecord job = jobWrapper.JobRecord;
job.Status = OperationStatus.Completed;
await _operationDataStore.UpdateReindexJobAsync(job, jobWrapper.ETag, CancellationToken.None);
ReindexJobWrapper updatedJobWrapper = await _operationDataStore.GetReindexJobByIdAsync(job.Id, CancellationToken.None);
ValidateReindexJobRecord(job, updatedJobWrapper?.JobRecord);
}
public async Task GivenAnOldVersionOfAReindexJob_WhenUpdatingTheReindexJob_ThenJobConflictExceptionShouldBeThrown()
{
ReindexJobWrapper jobWrapper = await CreateRunningReindexJob();
ReindexJobRecord job = jobWrapper.JobRecord;
// Update the job for a first time. This should not fail.
job.Status = OperationStatus.Completed;
WeakETag jobVersion = jobWrapper.ETag;
await _operationDataStore.UpdateReindexJobAsync(job, jobVersion, CancellationToken.None);
// Attempt to update the job a second time with the old version.
await Assert.ThrowsAsync <JobConflictException>(() => _operationDataStore.UpdateReindexJobAsync(job, jobVersion, CancellationToken.None));
}
public async Task GivenAGetRequest_WhenGettingAnExistingJob_ThenHttpResponseCodeShouldBeOk()
{
var request = new GetReindexRequest("id");
var jobRecord = new ReindexJobRecord("hash", 1, null);
var jobWrapper = new ReindexJobWrapper(jobRecord, WeakETag.FromVersionId("id"));
_fhirOperationDataStore.GetReindexJobByIdAsync("id", Arg.Any <CancellationToken>()).Returns(jobWrapper);
var handler = new GetReindexRequestHandler(_fhirOperationDataStore, DisabledFhirAuthorizationService.Instance);
var result = await handler.Handle(request, CancellationToken.None);
Assert.Equal(HttpStatusCode.OK, result.StatusCode);
}
public async Task GivenAGetRequest_WhenTooManyRequestsThrown_ThenTooManyRequestsThrown()
{
var request = new GetReindexRequest("id");
var jobRecord = new ReindexJobRecord(_resourceTypeSearchParameterHashMap, 1);
var jobWrapper = new ReindexJobWrapper(jobRecord, WeakETag.FromVersionId("id"));
_fhirOperationDataStore.GetReindexJobByIdAsync("id", CancellationToken.None).Throws(new Exception(null, new RequestRateExceededException(TimeSpan.FromMilliseconds(100))));
var handler = new GetReindexRequestHandler(_fhirOperationDataStore, DisabledFhirAuthorizationService.Instance);
Exception thrownException = await Assert.ThrowsAsync <Exception>(() => handler.Handle(request, CancellationToken.None));
Assert.IsType <RequestRateExceededException>(thrownException.InnerException);
}
public async Task GivenAGetRequest_WhenUserUnauthorized_ThenUnauthorizedFhirExceptionThrown()
{
var request = new GetReindexRequest("id");
var jobRecord = new ReindexJobRecord(_resourceTypeSearchParameterHashMap, new List <string>(), 1);
var jobWrapper = new ReindexJobWrapper(jobRecord, WeakETag.FromVersionId("id"));
_fhirOperationDataStore.GetReindexJobByIdAsync("id", Arg.Any <CancellationToken>()).Returns(jobWrapper);
var authorizationService = Substitute.For <IAuthorizationService <DataActions> >();
authorizationService.CheckAccess(DataActions.Reindex, Arg.Any <CancellationToken>()).Returns(DataActions.None);
var handler = new GetReindexRequestHandler(_fhirOperationDataStore, authorizationService);
await Assert.ThrowsAsync <UnauthorizedFhirActionException>(() => handler.Handle(request, CancellationToken.None));
}
public async Task GivenACancelRequest_WhenUserUnauthorized_ThenUnauthorizedFhirExceptionThrown()
{
var request = new CancelReindexRequest("id");
var jobRecord = new ReindexJobRecord("hash", 1, null);
var jobWrapper = new ReindexJobWrapper(jobRecord, WeakETag.FromVersionId("id"));
_fhirOperationDataStore.GetReindexJobByIdAsync("id", Arg.Any <CancellationToken>()).Returns(jobWrapper);
var authorizationService = Substitute.For <IFhirAuthorizationService>();
authorizationService.CheckAccess(DataActions.Reindex).Returns(DataActions.None);
var handler = new CancelReindexRequestHandler(_fhirOperationDataStore, authorizationService);
await Assert.ThrowsAsync <UnauthorizedFhirActionException>(() => handler.Handle(request, CancellationToken.None));
}
public async Task GivenACancelRequest_WhenJobCompleted_ThenRequestNotValidExceptionThrown()
{
var request = new CancelReindexRequest("id");
var jobRecord = new ReindexJobRecord("hash", 1, null)
{
Status = OperationStatus.Completed,
};
var jobWrapper = new ReindexJobWrapper(jobRecord, WeakETag.FromVersionId("id"));
_fhirOperationDataStore.GetReindexJobByIdAsync("id", Arg.Any <CancellationToken>()).Returns(jobWrapper);
var handler = new CancelReindexRequestHandler(_fhirOperationDataStore, DisabledFhirAuthorizationService.Instance);
await Assert.ThrowsAsync <RequestNotValidException>(() => handler.Handle(request, CancellationToken.None));
}
private async Task PerformReindexingOperation(CreateReindexResponse response, OperationStatus operationStatus, CancellationTokenSource cancellationTokenSource)
{
Task reindexWorkerTask = _reindexJobWorker.ExecuteAsync(cancellationTokenSource.Token);
ReindexJobWrapper reindexJobWrapper = await _fhirOperationDataStore.GetReindexJobByIdAsync(response.Job.JobRecord.Id, cancellationTokenSource.Token);
int delayCount = 0;
while (reindexJobWrapper.JobRecord.Status != operationStatus && delayCount < 10)
{
await Task.Delay(1000);
delayCount++;
reindexJobWrapper = await _fhirOperationDataStore.GetReindexJobByIdAsync(response.Job.JobRecord.Id, cancellationTokenSource.Token);
}
Assert.InRange(delayCount, 0, 9);
}
public static ResourceElement ToParametersResourceElement(this ReindexJobWrapper record)
{
EnsureArg.IsNotNull(record, nameof(record));
var parametersResource = new Parameters();
parametersResource.VersionId = record.ETag.VersionId;
var job = record.JobRecord;
var startTime = job.StartTime ?? DateTimeOffset.MinValue;
var endTime = job.StartTime ?? DateTimeOffset.MaxValue;
parametersResource.Parameter = new List <Parameters.ParameterComponent>();
parametersResource.Parameter.Add(new Parameters.ParameterComponent()
{
Name = JobRecordProperties.Id, Value = new FhirString(job.Id)
});
parametersResource.Parameter.Add(new Parameters.ParameterComponent()
{
Name = JobRecordProperties.StartTime, Value = new FhirDateTime(startTime)
});
parametersResource.Parameter.Add(new Parameters.ParameterComponent()
{
Name = JobRecordProperties.EndTime, Value = new FhirDateTime(endTime)
});
parametersResource.Parameter.Add(new Parameters.ParameterComponent()
{
Name = JobRecordProperties.Progress, Value = new FhirDecimal(job.PercentComplete)
});
parametersResource.Parameter.Add(new Parameters.ParameterComponent()
{
Name = JobRecordProperties.Status, Value = new FhirString(job.Status.ToString())
});
parametersResource.Parameter.Add(new Parameters.ParameterComponent()
{
Name = JobRecordProperties.MaximumConcurrency, Value = new FhirDecimal(job.MaximumConcurrency)
});
parametersResource.Parameter.Add(new Parameters.ParameterComponent()
{
Name = JobRecordProperties.Scope, Value = new FhirString(job.Scope)
});
return(parametersResource.ToResourceElement());
}
private async Task <ReindexJobWrapper> CreateRunningReindexJob()
{
// Create a queued job.
await InsertNewReindexJobRecordAsync();
// Acquire the job. This will timestamp it and set it to running.
IReadOnlyCollection <ReindexJobWrapper> jobWrappers = await AcquireReindexJobsAsync(maximumNumberOfConcurrentJobAllowed : 1);
Assert.NotNull(jobWrappers);
Assert.Equal(1, jobWrappers.Count);
ReindexJobWrapper jobWrapper = jobWrappers.FirstOrDefault();
Assert.NotNull(jobWrapper);
Assert.NotNull(jobWrapper.JobRecord);
Assert.Equal(OperationStatus.Running, jobWrapper.JobRecord.Status);
return(jobWrapper);
}
public async Task GivenACancelRequest_WhenJobInProgress_ThenJobUpdatedToCanceled()
{
var request = new CancelReindexRequest("id");
var jobRecord = new ReindexJobRecord("hash", 1, null)
{
Status = OperationStatus.Running,
};
var jobWrapper = new ReindexJobWrapper(jobRecord, WeakETag.FromVersionId("id"));
_fhirOperationDataStore.GetReindexJobByIdAsync("id", Arg.Any <CancellationToken>()).Returns(jobWrapper);
_fhirOperationDataStore.UpdateReindexJobAsync(jobRecord, WeakETag.FromVersionId("id"), Arg.Any <CancellationToken>()).Returns(jobWrapper);
var handler = new CancelReindexRequestHandler(_fhirOperationDataStore, DisabledFhirAuthorizationService.Instance);
var result = await handler.Handle(request, CancellationToken.None);
Assert.Equal(OperationStatus.Canceled, result.Job.JobRecord.Status);
}
