private async Task <JobViewModel> AddStartingProcessJobLog(string jobId) { if (string.IsNullOrWhiteSpace(jobId)) { _logger.Error($"No jobId given."); throw new NonRetriableException("No Job Id given"); } ApiResponse <JobViewModel> jobResponse = await _jobsApiClientPolicy.ExecuteAsync(() => _jobsApiClient.GetJobById(jobId)); if (jobResponse == null || jobResponse.Content == null) { _logger.Error($"Could not find the parent job with job id: '{jobId}'"); throw new NonRetriableException($"Could not find the parent job with job id: '{jobId}'"); } JobViewModel job = jobResponse.Content; if (job.CompletionStatus.HasValue) { _logger.Information($"Received job with id: '{job.Id}' is already in a completed state with status {job.CompletionStatus.ToString()}"); return(null); } await _jobsApiClientPolicy.ExecuteAsync(() => _jobsApiClient.AddJobLog(jobId, new JobLogUpdateModel())); return(job); }
public async Task ProcessDeadLetteredMessage_GivenMessageAndLogIsUpdated_LogsInformation() { // Arrange const string jobId = "job-id-1"; JobLog jobLog = new JobLog { Id = "job-log-id-1" }; ApiResponse <JobLog> jobLogResponse = new ApiResponse <JobLog>(HttpStatusCode.OK, jobLog); Message message = new Message(); message.UserProperties.Add("jobId", jobId); IJobsApiClient jobsApiClient = CreateJobsApiClient(); jobsApiClient .AddJobLog(Arg.Is(jobId), Arg.Any <JobLogUpdateModel>()) .Returns(jobLogResponse); ILogger logger = CreateLogger(); IJobHelperService service = CreateJobHelperService(jobsApiClient, logger); // Act await service.ProcessDeadLetteredMessage(message); // Assert logger .Received(1) .Information(Arg.Is($"A new job log was added to inform of a dead lettered message with job log id '{jobLog.Id}' on job with id '{jobId}'")); }
public async Task AddJobLog_Called_ReturnsJobLog() { string jobId = "5678"; IJobsApiClient jobsApiClient = Substitute.For <IJobsApiClient>(); JobManagementResiliencePolicies policies = new JobManagementResiliencePolicies { JobsApiClient = Policy.NoOpAsync() }; IMessengerService messengerService = Substitute.For <IMessengerService>(); ILogger logger = Substitute.For <ILogger>(); JobLog jobLog = new JobLog { JobId = jobId }; ApiResponse <JobLog> jobLogApiResponse = new ApiResponse <JobLog>(HttpStatusCode.OK, jobLog); JobLogUpdateModel jobLogUpdateModel = new JobLogUpdateModel(); jobsApiClient .AddJobLog(jobId, jobLogUpdateModel) .Returns(jobLogApiResponse); JobManagement jobManagement = new JobManagement(jobsApiClient, logger, policies, messengerService); //Act JobLog result = await jobManagement.AddJobLog(jobId, jobLogUpdateModel); Assert.AreEqual(result, jobLog); await jobsApiClient .Received(1) .AddJobLog(jobId, jobLogUpdateModel); }
public async Task UpdateJobStatus_ApiResponseSuccess_Runs() { //Arrange IJobsApiClient jobsApiClient = Substitute.For <IJobsApiClient>(); JobManagementResiliencePolicies policies = new JobManagementResiliencePolicies { JobsApiClient = Policy.NoOpAsync() }; IMessengerService messengerService = Substitute.For <IMessengerService>(); ILogger logger = Substitute.For <ILogger>(); ApiResponse <JobLog> jobLogApiResponse = new ApiResponse <JobLog>(HttpStatusCode.OK, new JobLog()); jobsApiClient .AddJobLog(Arg.Any <string>(), Arg.Any <JobLogUpdateModel>()) .Returns(jobLogApiResponse); JobLogUpdateModel updateModel = new JobLogUpdateModel(); JobManagement jobManagement = new JobManagement(jobsApiClient, logger, policies, messengerService); string jobId = "3456"; //Act await jobManagement.UpdateJobStatus(jobId, updateModel); //Assert await jobsApiClient .Received(1) .AddJobLog(jobId, updateModel); logger .Received(0) .Write(Arg.Any <LogEventLevel>(), Arg.Any <string>()); }
public async Task UpdateJobStatusInternal_ApiResponseFailure_Logs(ApiResponse <JobLog> jobLogApiResponse) { //Arrange IJobsApiClient jobsApiClient = Substitute.For <IJobsApiClient>(); JobManagementResiliencePolicies policies = new JobManagementResiliencePolicies { JobsApiClient = Policy.NoOpAsync() }; IMessengerService messengerService = Substitute.For <IMessengerService>(); ILogger logger = Substitute.For <ILogger>(); jobsApiClient .AddJobLog(Arg.Any <string>(), Arg.Any <JobLogUpdateModel>()) .Returns(jobLogApiResponse); JobLogUpdateModel updateModel = new JobLogUpdateModel(); JobManagement jobManagement = new JobManagement(jobsApiClient, logger, policies, messengerService); string jobId = "3456"; //Act await jobManagement.UpdateJobStatus(jobId, updateModel); //Assert await jobsApiClient .Received(1) .AddJobLog(jobId, updateModel); logger .Received(1) .Write(LogEventLevel.Error, $"Failed to add a job log for job id '{jobId}'"); }
private async Task UpdateJobStatus(string jobId, bool?completedSuccessfully, int percentComplete, string outcome = null) { JobLogUpdateModel jobLogUpdateModel = new JobLogUpdateModel { CompletedSuccessfully = completedSuccessfully, ItemsProcessed = percentComplete, Outcome = outcome }; ApiResponse <JobLog> jobLogResponse = await _jobsApiClientPolicy.ExecuteAsync(() => _jobsApiClient.AddJobLog(jobId, jobLogUpdateModel)); if (jobLogResponse == null || jobLogResponse.Content == null) { _logger.Error($"Failed to add a job log for job id '{jobId}'"); } }
public async Task UpdateJobStatus(string jobId, JobLogUpdateModel jobLogUpdateModel) { ApiResponse <JobLog> jobLogResponse = await _jobsApiClientPolicy.ExecuteAsync(() => _jobsApiClient.AddJobLog(jobId, jobLogUpdateModel)); if (jobLogResponse?.Content == null) { _logger.Write(LogEventLevel.Error, $"Failed to add a job log for job id '{jobId}'"); } }
public async Task UpdateAllocations(Message message) { Guard.ArgumentNotNull(message, nameof(message)); JobViewModel job = null; if (!message.UserProperties.ContainsKey("jobId")) { _logger.Error("Missing parent job id to instruct generating allocations"); return; } string jobId = message.UserProperties["jobId"].ToString(); ApiResponse <JobViewModel> response = await _jobsApiClientPolicy.ExecuteAsync(() => _jobsApiClient.GetJobById(jobId)); if (response == null || response.Content == null) { _logger.Error($"Could not find the parent job with job id: '{jobId}'"); throw new Exception($"Could not find the parent job with job id: '{jobId}'"); } job = response.Content; if (job.CompletionStatus.HasValue) { _logger.Information($"Received job with id: '{job.Id}' is already in a completed state with status {job.CompletionStatus.ToString()}"); return; } await _jobsApiClientPolicy.ExecuteAsync(() => _jobsApiClient.AddJobLog(jobId, new Common.ApiClient.Jobs.Models.JobLogUpdateModel())); IDictionary <string, string> properties = message.BuildMessageProperties(); string specificationId = message.UserProperties["specification-id"].ToString(); BuildProject buildProject = await GetBuildProjectForSpecificationId(specificationId); if (message.UserProperties.ContainsKey("ignore-save-provider-results")) { properties.Add("ignore-save-provider-results", "true"); } string cacheKey = ""; if (message.UserProperties.ContainsKey("provider-cache-key")) { cacheKey = message.UserProperties["provider-cache-key"].ToString(); } else { cacheKey = $"{CacheKeys.ScopedProviderSummariesPrefix}{specificationId}"; } bool summariesExist = await _cacheProvider.KeyExists <ProviderSummary>(cacheKey); long totalCount = await _cacheProvider.ListLengthAsync <ProviderSummary>(cacheKey); bool refreshCachedScopedProviders = false; if (summariesExist) { IEnumerable <string> scopedProviderIds = await _providerResultsRepository.GetScopedProviderIds(specificationId); if (scopedProviderIds.Count() != totalCount) { refreshCachedScopedProviders = true; } else { IEnumerable <ProviderSummary> cachedScopedSummaries = await _cacheProvider.ListRangeAsync <ProviderSummary>(cacheKey, 0, (int)totalCount); IEnumerable <string> differences = scopedProviderIds.Except(cachedScopedSummaries.Select(m => m.Id)); refreshCachedScopedProviders = differences.AnyWithNullCheck(); } } if (!summariesExist || refreshCachedScopedProviders) { totalCount = await _providerResultsRepository.PopulateProviderSummariesForSpecification(specificationId); } const string providerSummariesPartitionIndex = "provider-summaries-partition-index"; const string providerSummariesPartitionSize = "provider-summaries-partition-size"; properties.Add(providerSummariesPartitionSize, _engineSettings.MaxPartitionSize.ToString()); properties.Add("provider-cache-key", cacheKey); properties.Add("specification-id", specificationId); IList <IDictionary <string, string> > allJobProperties = new List <IDictionary <string, string> >(); for (int partitionIndex = 0; partitionIndex < totalCount; partitionIndex += _engineSettings.MaxPartitionSize) { if (properties.ContainsKey(providerSummariesPartitionIndex)) { properties[providerSummariesPartitionIndex] = partitionIndex.ToString(); } else { properties.Add(providerSummariesPartitionIndex, partitionIndex.ToString()); } IDictionary <string, string> jobProperties = new Dictionary <string, string>(); foreach (KeyValuePair <string, string> item in properties) { jobProperties.Add(item.Key, item.Value); } allJobProperties.Add(jobProperties); } await _specificationsRepository.UpdateCalculationLastUpdatedDate(specificationId); try { if (!allJobProperties.Any()) { _logger.Information($"No scoped providers set for specification '{specificationId}'"); JobLogUpdateModel jobCompletedLog = new JobLogUpdateModel { CompletedSuccessfully = true, Outcome = "Calculations not run as no scoped providers set for specification" }; await _jobsApiClientPolicy.ExecuteAsync(() => _jobsApiClient.AddJobLog(job.Id, jobCompletedLog)); return; } IEnumerable <Job> newJobs = await CreateGenerateAllocationJobs(job, allJobProperties); int newJobsCount = newJobs.Count(); int batchCount = allJobProperties.Count(); if (newJobsCount != batchCount) { throw new Exception($"Only {newJobsCount} child jobs from {batchCount} were created with parent id: '{job.Id}'"); } else { _logger.Information($"{newJobsCount} child jobs were created for parent id: '{job.Id}'"); } } catch (Exception ex) { _logger.Error(ex.Message); throw new Exception($"Failed to create child jobs for parent job: '{job.Id}'"); } }
public async Task ProcessDeadLetteredMessage(Message message) { Guard.ArgumentNotNull(message, nameof(message)); if (!message.UserProperties.ContainsKey("jobId")) { _logger.Error("Missing job id from dead lettered message"); return; } string jobId = message.UserProperties["jobId"].ToString(); Common.ApiClient.Jobs.Models.JobLogUpdateModel jobLogUpdateModel = new Common.ApiClient.Jobs.Models.JobLogUpdateModel { CompletedSuccessfully = false, Outcome = $"The job has exceeded its maximum retry count and failed to complete successfully" }; try { ApiResponse <Common.ApiClient.Jobs.Models.JobLog> jobLogResponse = await _jobsApiClientPolicy.ExecuteAsync(() => _jobsApiClient.AddJobLog(jobId, jobLogUpdateModel)); if (jobLogResponse == null || jobLogResponse.Content == null) { _logger.Error($"Failed to add a job log for job id '{jobId}'"); } else { _logger.Information($"A new job log was added to inform of a dead lettered message with job log id '{jobLogResponse.Content.Id}' on job with id '{jobId}'"); } } catch (Exception exception) { _logger.Error(exception, $"Failed to add a job log for job id '{jobId}'"); } }