public async Task <ReindexJobWrapper> CreateReindexJobAsync(ReindexJobRecord jobRecord, CancellationToken cancellationToken) { EnsureArg.IsNotNull(jobRecord, nameof(jobRecord)); var cosmosReindexJob = new CosmosReindexJobRecordWrapper(jobRecord); try { var result = await _containerScope.Value.CreateItemAsync( cosmosReindexJob, new PartitionKey(CosmosDbReindexConstants.ReindexJobPartitionKey), cancellationToken : cancellationToken); return(new ReindexJobWrapper(jobRecord, WeakETag.FromVersionId(result.Resource.ETag))); } catch (CosmosException dce) { if (dce.IsRequestRateExceeded()) { throw; } _logger.LogError(dce, "Failed to create a reindex job."); throw; } }
public async Task <CreateReindexResponse> Handle(CreateReindexRequest request, CancellationToken cancellationToken) { EnsureArg.IsNotNull(request, nameof(request)); if (await _authorizationService.CheckAccess(DataActions.Reindex) != DataActions.Reindex) { throw new UnauthorizedFhirActionException(); } if (await _fhirOperationDataStore.CheckActiveReindexJobsAsync(cancellationToken)) { throw new JobConflictException(Resources.OnlyOneResourceJobAllowed); } // TODO: determine new parameters to index // TODO: Get hash from parameters file string hash = "hash"; var jobRecord = new ReindexJobRecord( hash, request.MaximumConcurrency ?? _reindexJobConfiguration.DefaultMaximumThreadsPerReindexJob, request.Scope); var outcome = await _fhirOperationDataStore.CreateReindexJobAsync(jobRecord, cancellationToken); return(new CreateReindexResponse(outcome)); }
public async Task <ReindexJobWrapper> CreateReindexJobAsync(ReindexJobRecord jobRecord, CancellationToken cancellationToken) { EnsureArg.IsNotNull(jobRecord, nameof(jobRecord)); var cosmosReindexJob = new CosmosReindexJobRecordWrapper(jobRecord); try { ResourceResponse <Document> result = await _documentClientScope.Value.CreateDocumentAsync( CollectionUri, cosmosReindexJob, new RequestOptions() { PartitionKey = new PartitionKey(CosmosDbReindexConstants.ReindexJobPartitionKey) }, disableAutomaticIdGeneration : true, cancellationToken : cancellationToken); return(new ReindexJobWrapper(jobRecord, WeakETag.FromVersionId(result.Resource.ETag))); } catch (DocumentClientException dce) { if (dce.StatusCode == HttpStatusCode.TooManyRequests) { throw new RequestRateExceededException(dce.RetryAfter); } _logger.LogError(dce, "Failed to create a reindex job."); throw; } }
public async Task GivenATargetRUConsumption_WhenConsumedRUsDecreases_QueryDelayIsDecreased() { var throttleController = new ReindexJobCosmosThrottleController(_fhirRequestContextAccessor, new NullLogger <ReindexJobCosmosThrottleController>()); var reindexJob = new ReindexJobRecord(new Dictionary <string, string>(), targetDataStoreUsagePercentage: 80); reindexJob.QueryDelayIntervalInMilliseconds = 50; throttleController.Initialize(reindexJob, 1000); int loopCount = 0; while (loopCount < 17) { _output.WriteLine($"Current throttle based delay is: {throttleController.GetThrottleBasedDelay()}"); _fhirRequestContextAccessor.RequestContext.ResponseHeaders.Add(CosmosDbHeaders.RequestCharge, "100.0"); throttleController.UpdateDatastoreUsage(); await Task.Delay(reindexJob.QueryDelayIntervalInMilliseconds + throttleController.GetThrottleBasedDelay()); loopCount++; } loopCount = 0; while (loopCount < 17) { _output.WriteLine($"Current throttle based delay is: {throttleController.GetThrottleBasedDelay()}"); _fhirRequestContextAccessor.RequestContext.ResponseHeaders.Add(CosmosDbHeaders.RequestCharge, "10.0"); throttleController.UpdateDatastoreUsage(); await Task.Delay(reindexJob.QueryDelayIntervalInMilliseconds + throttleController.GetThrottleBasedDelay()); loopCount++; } _output.WriteLine($"Final throttle based delay is: {throttleController.GetThrottleBasedDelay()}"); Assert.Equal(0, throttleController.GetThrottleBasedDelay()); }
public CosmosReindexJobRecordWrapper(ReindexJobRecord reindexJobRecord) { EnsureArg.IsNotNull(reindexJobRecord, nameof(reindexJobRecord)); JobRecord = reindexJobRecord; Id = reindexJobRecord.Id; }
public async Task <CreateReindexResponse> Handle(CreateReindexRequest request, CancellationToken cancellationToken) { EnsureArg.IsNotNull(request, nameof(request)); if (await _authorizationService.CheckAccess(DataActions.Reindex, cancellationToken) != DataActions.Reindex) { throw new UnauthorizedFhirActionException(); } (var activeReindexJobs, var reindexJobId) = await _fhirOperationDataStore.CheckActiveReindexJobsAsync(cancellationToken); if (activeReindexJobs) { throw new JobConflictException(string.Format(Resources.OnlyOneResourceJobAllowed, reindexJobId)); } var jobRecord = new ReindexJobRecord( _searchParameterDefinitionManager.SearchParameterHashMap, request.MaximumConcurrency ?? _reindexJobConfiguration.DefaultMaximumThreadsPerReindexJob, request.MaximumResourcesPerQuery ?? _reindexJobConfiguration.MaximumNumberOfResourcesPerQuery, request.QueryDelayIntervalInMilliseconds ?? _reindexJobConfiguration.QueryDelayIntervalInMilliseconds, request.TargetDataStoreUsagePercentage); var outcome = await _fhirOperationDataStore.CreateReindexJobAsync(jobRecord, cancellationToken); return(new CreateReindexResponse(outcome)); }
public void Initialize(ReindexJobRecord reindexJobRecord, int?provisionedDatastoreCapacity) { EnsureArg.IsNotNull(reindexJobRecord, nameof(reindexJobRecord)); ReindexJobRecord = reindexJobRecord; _provisionedRUThroughput = provisionedDatastoreCapacity; _jobConfiguredBatchSize = reindexJobRecord.MaximumNumberOfResourcesPerQuery; if (ReindexJobRecord.TargetDataStoreUsagePercentage.HasValue && ReindexJobRecord.TargetDataStoreUsagePercentage.Value > 0 && _provisionedRUThroughput.HasValue && _provisionedRUThroughput > 0) { _targetRUs = _provisionedRUThroughput.Value * (ReindexJobRecord.TargetDataStoreUsagePercentage.Value / 100.0); _logger.LogInformation($"Reindex throttling initialized, target RUs: {_targetRUs}"); _delayFactor = 0; _rUsConsumedDuringInterval = 0.0; _initialized = true; _targetBatchSize = reindexJobRecord.MaximumNumberOfResourcesPerQuery; } else { _logger.LogInformation("Unable to initialize throttle controller. Throttling unavailable. Provisioned RUs: {0}", _provisionedRUThroughput); } }
public async Task <CreateReindexResponse> Handle(CreateReindexRequest request, CancellationToken cancellationToken) { EnsureArg.IsNotNull(request, nameof(request)); if (await _authorizationService.CheckAccess(DataActions.Reindex, cancellationToken) != DataActions.Reindex) { throw new UnauthorizedFhirActionException(); } (var activeReindexJobs, var reindexJobId) = await _fhirOperationDataStore.CheckActiveReindexJobsAsync(cancellationToken); if (activeReindexJobs) { throw new JobConflictException(string.Format(Resources.OnlyOneResourceJobAllowed, reindexJobId)); } // We need to pull in latest search parameter updates from the data store before creating a reindex job. // There could be a potential delay of <see cref="ReindexJobConfiguration.JobPollingFrequency"/> before // search parameter updates on one instance propagates to other instances. If we store the reindex // job with the old hash value in _searchParameterDefinitionManager.SearchParameterHashMap, then we will // not detect the resources that need to be reindexed. await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken); var jobRecord = new ReindexJobRecord( _searchParameterDefinitionManager.SearchParameterHashMap, request.MaximumConcurrency ?? _reindexJobConfiguration.DefaultMaximumThreadsPerReindexJob, request.MaximumResourcesPerQuery ?? _reindexJobConfiguration.MaximumNumberOfResourcesPerQuery, request.QueryDelayIntervalInMilliseconds ?? _reindexJobConfiguration.QueryDelayIntervalInMilliseconds, request.TargetDataStoreUsagePercentage); var outcome = await _fhirOperationDataStore.CreateReindexJobAsync(jobRecord, cancellationToken); return(new CreateReindexResponse(outcome)); }
public async Task GivenRunningJob_WhenExecuted_ThenQueuedQueryCompleted() { // Add one parameter that needs to be indexed var param = SearchParameterFixtureData.SearchDefinitionManager.AllSearchParameters.Where(p => p.Name == "appointment").FirstOrDefault(); param.IsSearchable = false; var job = new ReindexJobRecord(null, 1, null); // setup search result _searchService.SearchForReindexAsync( Arg.Any <IReadOnlyList <Tuple <string, string> > >(), Arg.Any <string>(), false, Arg.Any <CancellationToken>()). Returns(CreateSearchResult("token")); await _reindexJobTask.ExecuteAsync(job, _weakETag, _cancellationToken); // verify search for count await _searchService.Received().SearchForReindexAsync(Arg.Any <IReadOnlyList <Tuple <string, string> > >(), Arg.Any <string>(), true, Arg.Any <CancellationToken>()); // verify search for results await _searchService.Received().SearchForReindexAsync( Arg.Is <IReadOnlyList <Tuple <string, string> > >(l => l.Where(t => t.Item1 == "_type" && t.Item2 == "Appointment,AppointmentResponse").Any()), Arg.Any <string>(), false, Arg.Any <CancellationToken>()); Assert.Equal(OperationStatus.Running, job.Status); Assert.Equal(5, job.Count); Assert.Equal("Appointment,AppointmentResponse", job.ResourceList); Assert.Equal("appointment", job.SearchParamList); Assert.Collection <ReindexJobQueryStatus>( job.QueryList, item => Assert.True(item.ContinuationToken == null && item.Status == OperationStatus.Completed), item2 => Assert.True(item2.ContinuationToken == "token" && item2.Status == OperationStatus.Queued)); // setup search result _searchService.SearchForReindexAsync( Arg.Any <IReadOnlyList <Tuple <string, string> > >(), Arg.Any <string>(), false, Arg.Any <CancellationToken>()). Returns(CreateSearchResult(null)); await _reindexJobTask.ExecuteAsync(job, _weakETag, _cancellationToken); Assert.Equal(OperationStatus.Completed, job.Status); Assert.Equal(5, job.Count); Assert.Equal("Appointment,AppointmentResponse", job.ResourceList); Assert.Equal("appointment", job.SearchParamList); Assert.Collection <ReindexJobQueryStatus>( job.QueryList, item => Assert.True(item.ContinuationToken == null && item.Status == OperationStatus.Completed), item2 => Assert.True(item2.ContinuationToken == "token" && item2.Status == OperationStatus.Completed)); param.IsSearchable = true; }
public async Task GivenAMatchingReindexJob_WhenGettingById_ThenTheMatchingReindexJobShouldBeReturned() { ReindexJobRecord jobRecord = await InsertNewReindexJobRecordAsync(); ReindexJobWrapper jobWrapper = await _operationDataStore.GetReindexJobByIdAsync(jobRecord.Id, CancellationToken.None); Assert.Equal(jobRecord.Id, jobWrapper?.JobRecord?.Id); }
public async Task GivenNoSupportedParams_WhenExecuted_ThenJobCanceled() { var job = new ReindexJobRecord(null, 1, null); await _reindexJobTask.ExecuteAsync(job, _weakETag, _cancellationToken); Assert.Equal(OperationStatus.Canceled, job.Status); await _searchService.DidNotReceiveWithAnyArgs().SearchForReindexAsync(default, default, default, default);
private static GetReindexResponse GetReindexJobResponse() { var jobRecord = new ReindexJobRecord(_searchParameterHashMap, new List <string>(), 5); var jobWrapper = new ReindexJobWrapper( jobRecord, WeakETag.FromVersionId("33a64df551425fcc55e4d42a148795d9f25f89d4")); return(new GetReindexResponse(System.Net.HttpStatusCode.OK, jobWrapper)); }
private static GetReindexResponse GetReindexJobResponse() { var jobRecord = new ReindexJobRecord("hash", 5, "patient"); var jobWrapper = new ReindexJobWrapper( jobRecord, WeakETag.FromVersionId("33a64df551425fcc55e4d42a148795d9f25f89d4")); return(new GetReindexResponse(System.Net.HttpStatusCode.OK, jobWrapper)); }
private static CreateReindexResponse GetCreateReindexResponse() { var jobRecord = new ReindexJobRecord(_searchParameterHashMap, 5); var jobWrapper = new ReindexJobWrapper( jobRecord, WeakETag.FromVersionId("33a64df551425fcc55e4d42a148795d9f25f89d4")); return(new CreateReindexResponse(jobWrapper)); }
public async Task GivenNoReindexJobInQueuedState_WhenAcquiringReindexJobs_ThenNoReindexJobShouldBeReturned(OperationStatus operationStatus) { ReindexJobRecord jobRecord = await InsertNewReindexJobRecordAsync(jobRecord => jobRecord.Status = operationStatus); IReadOnlyCollection <ReindexJobWrapper> jobs = await AcquireReindexJobsAsync(); Assert.NotNull(jobs); Assert.Empty(jobs); }
private static CreateReindexResponse GetCreateReindexResponse() { var jobRecord = new ReindexJobRecord("hash", 5, "patient"); var jobWrapper = new ReindexJobWrapper( jobRecord, WeakETag.FromVersionId("33a64df551425fcc55e4d42a148795d9f25f89d4")); return(new CreateReindexResponse(jobWrapper)); }
public async Task GivenAnActiveReindexJob_WhenGettingActiveReindexJobs_ThenTheCorrectJobIdShouldBeReturned(OperationStatus operationStatus) { ReindexJobRecord jobRecord = await InsertNewReindexJobRecordAsync(job => job.Status = operationStatus); (bool, string)activeReindexJobResult = await _operationDataStore.CheckActiveReindexJobsAsync(CancellationToken.None); Assert.True(activeReindexJobResult.Item1); Assert.Equal(jobRecord.Id, activeReindexJobResult.Item2); }
/// <inheritdoc /> public async Task ExecuteAsync(ReindexJobRecord reindexJobRecord, WeakETag weakETag, CancellationToken cancellationToken) { EnsureArg.IsNotNull(reindexJobRecord, nameof(reindexJobRecord)); EnsureArg.IsNotNull(weakETag, nameof(weakETag)); _reindexJobRecord = reindexJobRecord; _weakETag = weakETag; try { // If we are resuming a job, we can detect that by checking the progress info from the job record. // If no queries have been added to the progress then this is a new job if (_reindexJobRecord.QueryList?.Count == 0) { // Build query based on new search params // Find supported, but not yet searchable params var notYetIndexedParams = _supportedSearchParameterDefinitionManager.GetSupportedButNotSearchableParams(); // From the param list, get the list of necessary resources which should be // included in our query var resourceList = new HashSet <string>(); foreach (var param in notYetIndexedParams) { resourceList.UnionWith(param.TargetResourceTypes); // TODO: Expand the BaseResourceTypes to all child resources resourceList.UnionWith(param.BaseResourceTypes); } _reindexJobRecord.Resources.AddRange(resourceList); _reindexJobRecord.SearchParams.AddRange(notYetIndexedParams.Select(p => p.Name)); } // This is just a shell for now, will be completed in future await CompleteJobAsync(OperationStatus.Completed, cancellationToken); _logger.LogTrace("Successfully completed the job."); } catch (JobConflictException) { // The reindex job was updated externally. _logger.LogTrace("The job was updated by another process."); } catch (Exception ex) { // The job has encountered an error it cannot recover from. // Try to update the job to failed state. _logger.LogError(ex, "Encountered an unhandled exception. The job will be marked as failed."); _reindexJobRecord.Error.Add(new OperationOutcomeIssue( OperationOutcomeConstants.IssueSeverity.Error, OperationOutcomeConstants.IssueType.Exception, ex.Message)); await CompleteJobAsync(OperationStatus.Failed, cancellationToken); } }
private async Task <ReindexJobRecord> InsertNewReindexJobRecordAsync(Action <ReindexJobRecord> jobRecordCustomizer = null) { var jobRecord = new ReindexJobRecord("searchParamHash", maxiumumConcurrency: 1, scope: "all"); jobRecordCustomizer?.Invoke(jobRecord); ReindexJobWrapper result = await _operationDataStore.CreateReindexJobAsync(jobRecord, CancellationToken.None); return(result.JobRecord); }
private void ValidateReindexJobRecord(ReindexJobRecord expected, ReindexJobRecord actual) { Assert.Equal(expected.Id, actual.Id); Assert.Equal(expected.CanceledTime, actual.CanceledTime); Assert.Equal(expected.EndTime, actual.EndTime); Assert.Equal(expected.Hash, actual.Hash); Assert.Equal(expected.SchemaVersion, actual.SchemaVersion); Assert.Equal(expected.StartTime, actual.StartTime); Assert.Equal(expected.Status, actual.Status); Assert.Equal(expected.QueuedTime, actual.QueuedTime); }
public async Task GivenANonexistentReindexJob_WhenUpdatingTheReindexJob_ThenJobNotFoundExceptionShouldBeThrown() { ReindexJobWrapper jobWrapper = await CreateRunningReindexJob(); ReindexJobRecord job = jobWrapper.JobRecord; WeakETag jobVersion = jobWrapper.ETag; await _testHelper.DeleteReindexJobRecordAsync(job.Id); await Assert.ThrowsAsync <JobNotFoundException>(() => _operationDataStore.UpdateReindexJobAsync(job, jobVersion, CancellationToken.None)); }
private void ValidateReindexJobRecord(ReindexJobRecord expected, ReindexJobRecord actual) { Assert.Equal(expected.Id, actual.Id); Assert.Equal(expected.CanceledTime, actual.CanceledTime); Assert.Equal(expected.EndTime, actual.EndTime); Assert.Equal(expected.ResourceTypeSearchParameterHashMap, actual.ResourceTypeSearchParameterHashMap); Assert.Equal(expected.SchemaVersion, actual.SchemaVersion); Assert.Equal(expected.StartTime, actual.StartTime); Assert.Equal(expected.Status, actual.Status); Assert.Equal(expected.QueuedTime, actual.QueuedTime); }
public void GivenThrottleControllerNotInitialized_WhenGetThrottleDelayCalled_ZeroReturned() { var throttleController = new ReindexJobCosmosThrottleController(_fhirRequestContextAccessor, new NullLogger <ReindexJobCosmosThrottleController>()); Assert.Equal(0, throttleController.GetThrottleBasedDelay()); var reindexJob = new ReindexJobRecord(new Dictionary <string, string>(), targetDataStoreUsagePercentage: null); reindexJob.QueryDelayIntervalInMilliseconds = 50; throttleController.Initialize(reindexJob, null); Assert.Equal(0, throttleController.GetThrottleBasedDelay()); }
public async Task <ReindexJobWrapper> UpdateReindexJobAsync(ReindexJobRecord jobRecord, WeakETag eTag, CancellationToken cancellationToken) { EnsureArg.IsNotNull(jobRecord, nameof(jobRecord)); var cosmosReindexJob = new CosmosReindexJobRecordWrapper(jobRecord); var requestOptions = new RequestOptions() { PartitionKey = new PartitionKey(CosmosDbReindexConstants.ReindexJobPartitionKey), }; // Create access condition so that record is replaced only if eTag matches. if (eTag != null) { requestOptions.AccessCondition = new AccessCondition() { Type = AccessConditionType.IfMatch, Condition = eTag.VersionId, }; } try { ResourceResponse <Document> replaceResult = await _retryExceptionPolicyFactory.CreateRetryPolicy().ExecuteAsync( () => _documentClientScope.Value.ReplaceDocumentAsync( UriFactory.CreateDocumentUri(DatabaseId, CollectionId, jobRecord.Id), cosmosReindexJob, requestOptions, cancellationToken)); var latestETag = replaceResult.Resource.ETag; return(new ReindexJobWrapper(jobRecord, WeakETag.FromVersionId(latestETag))); } catch (DocumentClientException dce) { if (dce.StatusCode == HttpStatusCode.TooManyRequests) { throw new RequestRateExceededException(dce.RetryAfter); } else if (dce.StatusCode == HttpStatusCode.PreconditionFailed) { throw new JobConflictException(); } else if (dce.StatusCode == HttpStatusCode.NotFound) { throw new JobNotFoundException(string.Format(Core.Resources.JobNotFound, jobRecord.Id)); } _logger.LogError(dce, "Failed to update a reindex job."); throw; } }
public async Task GivenAGetRequest_WhenTooManyRequestsThrown_ThenTooManyRequestsThrown() { var request = new GetReindexRequest("id"); var jobRecord = new ReindexJobRecord("hash", 1, null); var jobWrapper = new ReindexJobWrapper(jobRecord, WeakETag.FromVersionId("id")); _fhirOperationDataStore.GetReindexJobByIdAsync("id", CancellationToken.None).Throws(new RequestRateExceededException(TimeSpan.FromMilliseconds(100))); var handler = new GetReindexRequestHandler(_fhirOperationDataStore, DisabledFhirAuthorizationService.Instance); await Assert.ThrowsAsync <RequestRateExceededException>(() => handler.Handle(request, CancellationToken.None)); }
public async Task GivenAGetRequest_WhenIdNotFound_ThenJobNotFoundExceptionThrown() { var request = new GetReindexRequest("id"); var jobRecord = new ReindexJobRecord("hash", 1, null); var jobWrapper = new ReindexJobWrapper(jobRecord, WeakETag.FromVersionId("id")); _fhirOperationDataStore.GetReindexJobByIdAsync("id", Arg.Any <CancellationToken>()).Throws(new JobNotFoundException("not found")); var handler = new GetReindexRequestHandler(_fhirOperationDataStore, DisabledFhirAuthorizationService.Instance); await Assert.ThrowsAsync <JobNotFoundException>(() => handler.Handle(request, CancellationToken.None)); }
private async Task <ReindexJobRecord> InsertNewReindexJobRecordAsync(Action <ReindexJobRecord> jobRecordCustomizer = null) { Dictionary <string, string> searchParamHashMap = new Dictionary <string, string>(); searchParamHashMap.Add("Patient", "searchParamHash"); var jobRecord = new ReindexJobRecord(searchParamHashMap, maxiumumConcurrency: 1, scope: "all"); jobRecordCustomizer?.Invoke(jobRecord); ReindexJobWrapper result = await _operationDataStore.CreateReindexJobAsync(jobRecord, CancellationToken.None); return(result.JobRecord); }
public async Task GivenAnOldVersionOfAReindexJob_WhenUpdatingTheReindexJob_ThenJobConflictExceptionShouldBeThrown() { ReindexJobWrapper jobWrapper = await CreateRunningReindexJob(); ReindexJobRecord job = jobWrapper.JobRecord; // Update the job for a first time. This should not fail. job.Status = OperationStatus.Completed; WeakETag jobVersion = jobWrapper.ETag; await _operationDataStore.UpdateReindexJobAsync(job, jobVersion, CancellationToken.None); // Attempt to update the job a second time with the old version. await Assert.ThrowsAsync <JobConflictException>(() => _operationDataStore.UpdateReindexJobAsync(job, jobVersion, CancellationToken.None)); }
public async Task GivenThereIsNoRunningReindexJob_WhenAcquiringReindexJobs_ThenAvailableReindexJobsShouldBeReturned() { ReindexJobRecord jobRecord = await InsertNewReindexJobRecordAsync(); IReadOnlyCollection <ReindexJobWrapper> jobs = await AcquireReindexJobsAsync(); // The job should be marked as running now since it's acquired. jobRecord.Status = OperationStatus.Running; Assert.NotNull(jobs); Assert.Collection( jobs, job => ValidateReindexJobRecord(jobRecord, job.JobRecord)); }
public async Task GivenARunningReindexJob_WhenUpdatingTheReindexJob_ThenTheJobShouldBeUpdated() { ReindexJobWrapper jobWrapper = await CreateRunningReindexJob(); ReindexJobRecord job = jobWrapper.JobRecord; job.Status = OperationStatus.Completed; await _operationDataStore.UpdateReindexJobAsync(job, jobWrapper.ETag, CancellationToken.None); ReindexJobWrapper updatedJobWrapper = await _operationDataStore.GetReindexJobByIdAsync(job.Id, CancellationToken.None); ValidateReindexJobRecord(job, updatedJobWrapper?.JobRecord); }