public void QueueVacancyPages(StorageQueueMessage scheduledQueueMessage) { _logger.Debug("Retrieving vacancy summary page count"); var vacancyPageCount = _vacancyIndexDataProvider.GetVacancyPageCount(); _logger.Info("Retrieved vacancy summary page count of {0}", vacancyPageCount); if (vacancyPageCount == 0) { _logger.Warn("Expected vacancy page count to be greater than zero. Indexes will not be created successfully"); _processControlQueue.DeleteMessage(scheduledQueueMessage.MessageId, scheduledQueueMessage.PopReceipt); return; } var vacancySumaries = BuildVacancySummaryPages(scheduledQueueMessage.ExpectedExecutionTime, vacancyPageCount).ToList(); foreach (var vacancySummaryPage in vacancySumaries) { _messageBus.PublishMessage(vacancySummaryPage); } // Only delete from queue once we have all vacancies from the service without error. _processControlQueue.DeleteMessage(scheduledQueueMessage.MessageId, scheduledQueueMessage.PopReceipt); _logger.Info("Queued {0} vacancy summary pages", vacancySumaries.Count()); }
public void ShouldReturnThePageCountForVacancies() { var result = _vacancyIndexDataProvider.GetVacancyPageCount(); // Assert. result.Should().BePositive(); }
public void ProcessVacancyPages(StorageQueueMessage scheduledQueueMessage) { _logger.Debug("Retrieving vacancy summary page count"); var vacancyPageCount = _vacancyIndexDataProvider.GetVacancyPageCount(); _logger.Info("Retrieved vacancy summary page count of {0}", vacancyPageCount); if (vacancyPageCount == 0) { _logger.Warn("Expected vacancy page count to be greater than zero. Indexes will not be created successfully"); _jobControlQueue.DeleteMessage(ScheduledJobQueues.VacancyEtl, scheduledQueueMessage.MessageId, scheduledQueueMessage.PopReceipt); return; } var vacancySummaries = BuildVacancySummaryPages(scheduledQueueMessage.ExpectedExecutionTime, vacancyPageCount).ToList(); // Only delete from queue once we have all vacancies from the service without error. _jobControlQueue.DeleteMessage(ScheduledJobQueues.VacancyEtl, scheduledQueueMessage.MessageId, scheduledQueueMessage.PopReceipt); //Process pages Parallel.ForEach(vacancySummaries, new ParallelOptions { MaxDegreeOfParallelism = 1 }, ProcessVacancySummaryPage); var lastVacancySummaryPage = vacancySummaries.Last(); _logger.Info("Vacancy ETL Queue completed: {0} vacancy summary pages processed ", lastVacancySummaryPage.TotalPages); _logger.Info("Publishing VacancySummaryUpdateComplete message to queue"); var vsuc = new VacancySummaryUpdateComplete { ScheduledRefreshDateTime = lastVacancySummaryPage.ScheduledRefreshDateTime }; _serviceBus.PublishMessage(vsuc); _logger.Info("Published VacancySummaryUpdateComplete message published to queue"); }