public void QueueVacancyPages(StorageQueueMessage scheduledQueueMessage)
{
_logger.Debug("Retrieving vacancy summary page count");
var vacancyPageCount = _vacancyIndexDataProvider.GetVacancyPageCount();
_logger.Info("Retrieved vacancy summary page count of {0}", vacancyPageCount);
if (vacancyPageCount == 0)
{
_logger.Warn("Expected vacancy page count to be greater than zero. Indexes will not be created successfully");
_processControlQueue.DeleteMessage(scheduledQueueMessage.MessageId, scheduledQueueMessage.PopReceipt);
return;
}
var vacancySumaries = BuildVacancySummaryPages(scheduledQueueMessage.ExpectedExecutionTime, vacancyPageCount).ToList();
foreach (var vacancySummaryPage in vacancySumaries)
{
_messageBus.PublishMessage(vacancySummaryPage);
}
// Only delete from queue once we have all vacancies from the service without error.
_processControlQueue.DeleteMessage(scheduledQueueMessage.MessageId, scheduledQueueMessage.PopReceipt);
_logger.Info("Queued {0} vacancy summary pages", vacancySumaries.Count());
}
public void ShouldReturnThePageCountForVacancies()
{
var result = _vacancyIndexDataProvider.GetVacancyPageCount();
// Assert.
result.Should().BePositive();
}
public void ProcessVacancyPages(StorageQueueMessage scheduledQueueMessage)
{
_logger.Debug("Retrieving vacancy summary page count");
var vacancyPageCount = _vacancyIndexDataProvider.GetVacancyPageCount();
_logger.Info("Retrieved vacancy summary page count of {0}", vacancyPageCount);
if (vacancyPageCount == 0)
{
_logger.Warn("Expected vacancy page count to be greater than zero. Indexes will not be created successfully");
_jobControlQueue.DeleteMessage(ScheduledJobQueues.VacancyEtl, scheduledQueueMessage.MessageId, scheduledQueueMessage.PopReceipt);
return;
}
var vacancySummaries = BuildVacancySummaryPages(scheduledQueueMessage.ExpectedExecutionTime, vacancyPageCount).ToList();
// Only delete from queue once we have all vacancies from the service without error.
_jobControlQueue.DeleteMessage(ScheduledJobQueues.VacancyEtl, scheduledQueueMessage.MessageId, scheduledQueueMessage.PopReceipt);
//Process pages
Parallel.ForEach(vacancySummaries, new ParallelOptions {
MaxDegreeOfParallelism = 1
}, ProcessVacancySummaryPage);
var lastVacancySummaryPage = vacancySummaries.Last();
_logger.Info("Vacancy ETL Queue completed: {0} vacancy summary pages processed ", lastVacancySummaryPage.TotalPages);
_logger.Info("Publishing VacancySummaryUpdateComplete message to queue");
var vsuc = new VacancySummaryUpdateComplete
{
ScheduledRefreshDateTime = lastVacancySummaryPage.ScheduledRefreshDateTime
};
_serviceBus.PublishMessage(vsuc);
_logger.Info("Published VacancySummaryUpdateComplete message published to queue");
}