private void ConsumeTask(VacancySummaryPage vacancySummaryPage) { _vacancySummaryProcessor.QueueVacancySummaries(vacancySummaryPage); if (vacancySummaryPage.PageNumber == vacancySummaryPage.TotalPages) { _logger.Info("Vacancy ETL Queue completed: {0} vacancy summary pages queued ", vacancySummaryPage.TotalPages); _logger.Info("Publishing VacancySummaryUpdateComplete message to queue"); var vsuc = new VacancySummaryUpdateComplete { ScheduledRefreshDateTime = vacancySummaryPage.ScheduledRefreshDateTime }; _messageBus.PublishMessage(vsuc); _logger.Info("Published VacancySummaryUpdateComplete message published to queue"); } }
public void ProcessVacancyPages(StorageQueueMessage scheduledQueueMessage) { _logger.Debug("Retrieving vacancy summary page count"); var vacancyPageCount = _vacancyIndexDataProvider.GetVacancyPageCount(); _logger.Info("Retrieved vacancy summary page count of {0}", vacancyPageCount); if (vacancyPageCount == 0) { _logger.Warn("Expected vacancy page count to be greater than zero. Indexes will not be created successfully"); _jobControlQueue.DeleteMessage(ScheduledJobQueues.VacancyEtl, scheduledQueueMessage.MessageId, scheduledQueueMessage.PopReceipt); return; } var vacancySummaries = BuildVacancySummaryPages(scheduledQueueMessage.ExpectedExecutionTime, vacancyPageCount).ToList(); // Only delete from queue once we have all vacancies from the service without error. _jobControlQueue.DeleteMessage(ScheduledJobQueues.VacancyEtl, scheduledQueueMessage.MessageId, scheduledQueueMessage.PopReceipt); //Process pages Parallel.ForEach(vacancySummaries, new ParallelOptions { MaxDegreeOfParallelism = 1 }, ProcessVacancySummaryPage); var lastVacancySummaryPage = vacancySummaries.Last(); _logger.Info("Vacancy ETL Queue completed: {0} vacancy summary pages processed ", lastVacancySummaryPage.TotalPages); _logger.Info("Publishing VacancySummaryUpdateComplete message to queue"); var vsuc = new VacancySummaryUpdateComplete { ScheduledRefreshDateTime = lastVacancySummaryPage.ScheduledRefreshDateTime }; _serviceBus.PublishMessage(vsuc); _logger.Info("Published VacancySummaryUpdateComplete message published to queue"); }