private void ConsumeTask(VacancySummaryPage vacancySummaryPage)
{
_vacancySummaryProcessor.QueueVacancySummaries(vacancySummaryPage);
if (vacancySummaryPage.PageNumber == vacancySummaryPage.TotalPages)
{
_logger.Info("Vacancy ETL Queue completed: {0} vacancy summary pages queued ", vacancySummaryPage.TotalPages);
_logger.Info("Publishing VacancySummaryUpdateComplete message to queue");
var vsuc = new VacancySummaryUpdateComplete
{
ScheduledRefreshDateTime = vacancySummaryPage.ScheduledRefreshDateTime
};
_messageBus.PublishMessage(vsuc);
_logger.Info("Published VacancySummaryUpdateComplete message published to queue");
}
}
public void ProcessVacancyPages(StorageQueueMessage scheduledQueueMessage)
{
_logger.Debug("Retrieving vacancy summary page count");
var vacancyPageCount = _vacancyIndexDataProvider.GetVacancyPageCount();
_logger.Info("Retrieved vacancy summary page count of {0}", vacancyPageCount);
if (vacancyPageCount == 0)
{
_logger.Warn("Expected vacancy page count to be greater than zero. Indexes will not be created successfully");
_jobControlQueue.DeleteMessage(ScheduledJobQueues.VacancyEtl, scheduledQueueMessage.MessageId, scheduledQueueMessage.PopReceipt);
return;
}
var vacancySummaries = BuildVacancySummaryPages(scheduledQueueMessage.ExpectedExecutionTime, vacancyPageCount).ToList();
// Only delete from queue once we have all vacancies from the service without error.
_jobControlQueue.DeleteMessage(ScheduledJobQueues.VacancyEtl, scheduledQueueMessage.MessageId, scheduledQueueMessage.PopReceipt);
//Process pages
Parallel.ForEach(vacancySummaries, new ParallelOptions {
MaxDegreeOfParallelism = 1
}, ProcessVacancySummaryPage);
var lastVacancySummaryPage = vacancySummaries.Last();
_logger.Info("Vacancy ETL Queue completed: {0} vacancy summary pages processed ", lastVacancySummaryPage.TotalPages);
_logger.Info("Publishing VacancySummaryUpdateComplete message to queue");
var vsuc = new VacancySummaryUpdateComplete
{
ScheduledRefreshDateTime = lastVacancySummaryPage.ScheduledRefreshDateTime
};
_serviceBus.PublishMessage(vsuc);
_logger.Info("Published VacancySummaryUpdateComplete message published to queue");
}
