public async Task PopulateIndexAsync(bool units)
{
var indexClient = GetIndexClient();
using (var context = _contextFactory.GetLarsContext())
{
// For an unknown reason large datasets with complex joins appear to randomly
// drop data. Not sure why but advised approach is to preload data and query
// that rather than do all in single EF query.
var academicYears = _academicYearService.GetAcademicYears(context);
var issuingAuthorities = await _issuingAuthorityService.GetIssuingAuthoritiesAsync(context);
var componentTypes = await _componentTypeService.GetComponentTypesAsync(context);
var fundings = await _fundingService.GetFundingsAsync(context);
var categories = await _learningDeliveryCategoryService.GetLearningDeliveryCategoriesAsync(context);
var validities = await _validityService.GetValiditiesAsync(context);
var entitlementCategories = await _entitlementCategoryService.GetEntitlementCategoriesAsync(context);
var frameworkAims = await _frameworkAimService.GetLearningAimFrameworkAimsAsync(context, units);
var awardBodyCodes = await _awardOrgService.GetAwardingOrgNamesAsync(context);
var page = 0;
var next = true;
while (next)
{
var queryStartTime = DateTime.Now;
var learningAimsQueryable = context.LarsLearningDeliveries.AsQueryable();
if (units)
{
learningAimsQueryable = learningAimsQueryable.Where(ld => ld.LearnAimRefType == UnitLearnAimRefType);
}
else
{
learningAimsQueryable = learningAimsQueryable.Where(ld => ld.LearnAimRefType != UnitLearnAimRefType);
}
var learningAims = learningAimsQueryable
.OrderBy(ld => ld.LearnAimRef)
.ThenBy(ld => ld.EffectiveFrom)
.Skip(page * PageSize)
.Take(PageSize)
.Select(ld => new LearningAimModel
{
LearnAimRef = ld.LearnAimRef,
AwardingBodyCode = ld.AwardOrgCode,
EffectiveFrom = ld.EffectiveFrom,
EffectiveTo = ld.EffectiveTo,
Level = ld.NotionalNvqlevelv2Navigation.NotionalNvqlevelV2,
LevelDescription = ld.NotionalNvqlevelv2Navigation.NotionalNvqlevelV2desc,
Type = ld.LearnAimRefTypeNavigation.LearnAimRefTypeDesc,
LearningAimTitle = ld.LearnAimRefTitle,
GuidedLearningHours = ld.GuidedLearningHours.ToString(),
IsOFQUAL = ld.LearningDeliveryGenre == OFQUALRegulatedQualification ||
ld.LearningDeliveryGenre == OFQUALRegulatedUnit
})
.ToArray();
var queryEndTime = DateTime.Now;
var postProcessStartTime = DateTime.Now;
foreach (var learningDelivery in learningAims)
{
PopulateFrameworks(learningDelivery, frameworkAims, issuingAuthorities, componentTypes);
PopulateCategories(learningDelivery, categories.GetValueOrDefault(learningDelivery.LearnAimRef, new List <CategoryModel>()));
learningDelivery.AwardingBodyName = awardBodyCodes.GetValueOrDefault(learningDelivery.AwardingBodyCode);
learningDelivery.GuidedLearningHours = GetGuidedLearningHours(learningDelivery.GuidedLearningHours);
var fundingForDelivery = fundings.GetValueOrDefault(learningDelivery.LearnAimRef, new List <FundingModel>());
var validityForDelivery = validities.GetValueOrDefault(learningDelivery.LearnAimRef, new List <ValidityModel>());
var entitlementCategory = entitlementCategories.GetValueOrDefault(learningDelivery.LearnAimRef, new List <EntitlementCategoryModel>());
PopulateAcademicYears(learningDelivery, academicYears.ToList(), fundingForDelivery, entitlementCategory, validityForDelivery);
}
var postProcessEndTime = DateTime.Now;
var indexActions = learningAims.Select(IndexAction.Upload);
var batch = IndexBatch.New(indexActions);
if (batch.Actions.Any())
{
var startTime = DateTime.Now;
indexClient.Documents.Index(batch);
var endTime = DateTime.Now;
var duration = queryEndTime - queryStartTime;
page++;
Console.WriteLine($"Processed {page * PageSize} learning aim documents");
Console.WriteLine($"query time: {duration.Minutes} mins, {duration.Seconds} secs, {duration.Milliseconds} ms)");
duration = postProcessEndTime - postProcessStartTime;
Console.WriteLine($"post process time: {duration.Minutes} mins, {duration.Seconds} secs, {duration.Milliseconds} ms)");
duration = endTime - startTime;
Console.WriteLine($"batch time: {duration.Minutes} mins, {duration.Seconds} secs, {duration.Milliseconds} ms) \n");
}
else
{
next = false;
}
}
}
Console.WriteLine("Waiting for indexing...\n");
Thread.Sleep(2000);
}
