public async Task PopulateIndexAsync(bool units)
{
var indexClient = GetIndexClient();
using (var context = _contextFactory.GetLarsContext())
{
// For an unknown reason large datasets with complex joins appear to randomly
// drop data. Not sure why but advised approach is to preload data and query
// that rather than do all in single EF query.
var academicYears = _academicYearService.GetAcademicYears(context);
var issuingAuthorities = await _issuingAuthorityService.GetIssuingAuthoritiesAsync(context);
var componentTypes = await _componentTypeService.GetComponentTypesAsync(context);
var fundings = await _fundingService.GetFundingsAsync(context);
var categories = await _learningDeliveryCategoryService.GetLearningDeliveryCategoriesAsync(context);
var validities = await _validityService.GetValiditiesAsync(context);
var entitlementCategories = await _entitlementCategoryService.GetEntitlementCategoriesAsync(context);
var frameworkAims = await _frameworkAimService.GetLearningAimFrameworkAimsAsync(context, units);
var awardBodyCodes = await _awardOrgService.GetAwardingOrgNamesAsync(context);
var page = 0;
var next = true;
while (next)
{
var queryStartTime = DateTime.Now;
var learningAimsQueryable = context.LarsLearningDeliveries.AsQueryable();
if (units)
{
learningAimsQueryable = learningAimsQueryable.Where(ld => ld.LearnAimRefType == UnitLearnAimRefType);
}
else
{
learningAimsQueryable = learningAimsQueryable.Where(ld => ld.LearnAimRefType != UnitLearnAimRefType);
}
var learningAims = learningAimsQueryable
.OrderBy(ld => ld.LearnAimRef)
.ThenBy(ld => ld.EffectiveFrom)
.Skip(page * PageSize)
.Take(PageSize)
.Select(ld => new LearningAimModel
{
LearnAimRef = ld.LearnAimRef,
AwardingBodyCode = ld.AwardOrgCode,
EffectiveFrom = ld.EffectiveFrom,
EffectiveTo = ld.EffectiveTo,
Level = ld.NotionalNvqlevelv2Navigation.NotionalNvqlevelV2,
LevelDescription = ld.NotionalNvqlevelv2Navigation.NotionalNvqlevelV2desc,
Type = ld.LearnAimRefTypeNavigation.LearnAimRefTypeDesc,
LearningAimTitle = ld.LearnAimRefTitle,
GuidedLearningHours = ld.GuidedLearningHours.ToString(),
IsOFQUAL = ld.LearningDeliveryGenre == OFQUALRegulatedQualification ||
ld.LearningDeliveryGenre == OFQUALRegulatedUnit
})
.ToArray();
var queryEndTime = DateTime.Now;
var postProcessStartTime = DateTime.Now;
foreach (var learningDelivery in learningAims)
{
PopulateFrameworks(learningDelivery, frameworkAims, issuingAuthorities, componentTypes);
PopulateCategories(learningDelivery, categories.GetValueOrDefault(learningDelivery.LearnAimRef, new List <CategoryModel>()));
learningDelivery.AwardingBodyName = awardBodyCodes.GetValueOrDefault(learningDelivery.AwardingBodyCode);
learningDelivery.GuidedLearningHours = GetGuidedLearningHours(learningDelivery.GuidedLearningHours);
var fundingForDelivery = fundings.GetValueOrDefault(learningDelivery.LearnAimRef, new List <FundingModel>());
var validityForDelivery = validities.GetValueOrDefault(learningDelivery.LearnAimRef, new List <ValidityModel>());
var entitlementCategory = entitlementCategories.GetValueOrDefault(learningDelivery.LearnAimRef, new List <EntitlementCategoryModel>());
PopulateAcademicYears(learningDelivery, academicYears.ToList(), fundingForDelivery, entitlementCategory, validityForDelivery);
}
var postProcessEndTime = DateTime.Now;
var indexActions = learningAims.Select(IndexAction.Upload);
var batch = IndexBatch.New(indexActions);
if (batch.Actions.Any())
{
var startTime = DateTime.Now;
indexClient.Documents.Index(batch);
var endTime = DateTime.Now;
var duration = queryEndTime - queryStartTime;
page++;
Console.WriteLine($"Processed {page * PageSize} learning aim documents");
Console.WriteLine($"query time: {duration.Minutes} mins, {duration.Seconds} secs, {duration.Milliseconds} ms)");
duration = postProcessEndTime - postProcessStartTime;
Console.WriteLine($"post process time: {duration.Minutes} mins, {duration.Seconds} secs, {duration.Milliseconds} ms)");
duration = endTime - startTime;
Console.WriteLine($"batch time: {duration.Minutes} mins, {duration.Seconds} secs, {duration.Milliseconds} ms) \n");
}
else
{
next = false;
}
}
}
Console.WriteLine("Waiting for indexing...\n");
Thread.Sleep(2000);
}
public async override Task PopulateIndexAsync(CancellationToken cancellationToken)
{
var indexClient = GetIndexClient();
LookUpModel lookups;
using (var context = _contextFactory.GetLarsContext())
{
lookups = new LookUpModel
{
LookUpKey = "1",
NotionalNvqLevel2Lookups = await context.LarsNotionalNvqlevelv2Lookups
.Select(lvl => new NotionalNVQLevel2LookupModel
{
NotionalNVQLevelV2 = lvl.NotionalNvqlevelV2,
NotionalNVQLevelV2Desc = lvl.NotionalNvqlevelV2desc
}).ToListAsync(cancellationToken),
AcademicYearLookups = _academicYearService.GetAcademicYears(context)
.Select(ay => new AcademicYearLookupModel
{
IsCurrentAcademicYear = _academicYearService.IsCurrentAcademicYear(ay),
AcademicYear = ay.AcademicYear,
AcademicYearDesc = _academicYearService.FormatDescription(ay.AcademicYearDesc)
}).ToList(),
AwardingBodyLookups = await context.LarsAwardOrgCodeLookups
.Select(ab => new AwardingBodyLookupModel
{
AwardingBodyCode = ab.AwardOrgCode,
AwardingBodyName = ab.AwardOrgName
}).ToListAsync(cancellationToken),
ValidityCategoryLookups = await context.LarsValidityCategoryLookups
.Select(vc => new ValidityCategoryLookupModel
{
ValidityCategory = vc.ValidityCategory,
ValidityCategoryDescription = vc.ValidityCategoryDesc2
}).ToListAsync(cancellationToken),
ValidityFundingMappingLookups = await context.LarsValidityFundingMappings
.Select(fm => new ValidityFundingMappingLookupModel
{
ValidityCategory = fm.ValidityCategory,
FundingCategory = fm.FundingCategory,
EffectiveFrom = fm.EffectiveFrom,
EffectiveTo = fm.EffectiveTo
}).ToListAsync(cancellationToken),
FrameworkTypeLookups = await context.LarsProgTypeLookups
.Where(ft => !_tlevelProgTypes.Contains(ft.ProgType))
.Select(ft => new FrameworkTypeLookupModel
{
FrameworkType = ft.ProgType.ToString(),
FrameworkTypeDesc = ft.ProgTypeDesc
}).ToListAsync(cancellationToken),
TLevelTypeLookups = await context.LarsProgTypeLookups
.Where(ft => _tlevelProgTypes.Contains(ft.ProgType))
.Select(ft => new FrameworkTypeLookupModel
{
FrameworkType = ft.ProgType.ToString(),
FrameworkTypeDesc = ft.ProgTypeDesc
}).ToListAsync(cancellationToken),
IssuingAuthorityLookups = await context.LarsIssuingAuthorityLookups
.Select(ia => new IssuingAuthorityLookupModel
{
IssuingAuthority = ia.IssuingAuthority.ToString(),
IssuingAuthorityDesc = ia.IssuingAuthorityDesc
}).ToListAsync(cancellationToken),
StandardSectorLookups = await context.LarsStandardSectorCodeLookups
.Select(sc => new StandardSectorLookupModel
{
StandardSectorCode = sc.StandardSectorCode,
StandardSectorCodeDesc = sc.StandardSectorCodeDesc2
}).ToListAsync(cancellationToken),
SectorSubjectAreaTier1Lookups = await context.LarsSectorSubjectAreaTier1Lookups
.Select(st => new SectorSubjectAreaTier1LookupModel
{
SectorSubjectAreaTier1 = st.SectorSubjectAreaTier1.ToString(),
SectorSubjectAreaTier1Desc = st.SectorSubjectAreaTier1Desc
}).ToListAsync(cancellationToken)
};
}
Sort(lookups);
var indexActions = new List <IndexAction <LookUpModel> > {
IndexAction.Upload(lookups)
};
var batch = IndexBatch.New(indexActions);
if (batch.Actions.Any())
{
indexClient.Documents.Index(batch);
}
}
