public async Task PopulateIndexAsync(bool units) { var indexClient = GetIndexClient(); using (var context = _contextFactory.GetLarsContext()) { // For an unknown reason large datasets with complex joins appear to randomly // drop data. Not sure why but advised approach is to preload data and query // that rather than do all in single EF query. var academicYears = _academicYearService.GetAcademicYears(context); var issuingAuthorities = await _issuingAuthorityService.GetIssuingAuthoritiesAsync(context); var componentTypes = await _componentTypeService.GetComponentTypesAsync(context); var fundings = await _fundingService.GetFundingsAsync(context); var categories = await _learningDeliveryCategoryService.GetLearningDeliveryCategoriesAsync(context); var validities = await _validityService.GetValiditiesAsync(context); var entitlementCategories = await _entitlementCategoryService.GetEntitlementCategoriesAsync(context); var frameworkAims = await _frameworkAimService.GetLearningAimFrameworkAimsAsync(context, units); var awardBodyCodes = await _awardOrgService.GetAwardingOrgNamesAsync(context); var page = 0; var next = true; while (next) { var queryStartTime = DateTime.Now; var learningAimsQueryable = context.LarsLearningDeliveries.AsQueryable(); if (units) { learningAimsQueryable = learningAimsQueryable.Where(ld => ld.LearnAimRefType == UnitLearnAimRefType); } else { learningAimsQueryable = learningAimsQueryable.Where(ld => ld.LearnAimRefType != UnitLearnAimRefType); } var learningAims = learningAimsQueryable .OrderBy(ld => ld.LearnAimRef) .ThenBy(ld => ld.EffectiveFrom) .Skip(page * PageSize) .Take(PageSize) .Select(ld => new LearningAimModel { LearnAimRef = ld.LearnAimRef, AwardingBodyCode = ld.AwardOrgCode, EffectiveFrom = ld.EffectiveFrom, EffectiveTo = ld.EffectiveTo, Level = ld.NotionalNvqlevelv2Navigation.NotionalNvqlevelV2, LevelDescription = ld.NotionalNvqlevelv2Navigation.NotionalNvqlevelV2desc, Type = ld.LearnAimRefTypeNavigation.LearnAimRefTypeDesc, LearningAimTitle = ld.LearnAimRefTitle, GuidedLearningHours = ld.GuidedLearningHours.ToString(), IsOFQUAL = ld.LearningDeliveryGenre == OFQUALRegulatedQualification || ld.LearningDeliveryGenre == OFQUALRegulatedUnit }) .ToArray(); var queryEndTime = DateTime.Now; var postProcessStartTime = DateTime.Now; foreach (var learningDelivery in learningAims) { PopulateFrameworks(learningDelivery, frameworkAims, issuingAuthorities, componentTypes); PopulateCategories(learningDelivery, categories.GetValueOrDefault(learningDelivery.LearnAimRef, new List <CategoryModel>())); learningDelivery.AwardingBodyName = awardBodyCodes.GetValueOrDefault(learningDelivery.AwardingBodyCode); learningDelivery.GuidedLearningHours = GetGuidedLearningHours(learningDelivery.GuidedLearningHours); var fundingForDelivery = fundings.GetValueOrDefault(learningDelivery.LearnAimRef, new List <FundingModel>()); var validityForDelivery = validities.GetValueOrDefault(learningDelivery.LearnAimRef, new List <ValidityModel>()); var entitlementCategory = entitlementCategories.GetValueOrDefault(learningDelivery.LearnAimRef, new List <EntitlementCategoryModel>()); PopulateAcademicYears(learningDelivery, academicYears.ToList(), fundingForDelivery, entitlementCategory, validityForDelivery); } var postProcessEndTime = DateTime.Now; var indexActions = learningAims.Select(IndexAction.Upload); var batch = IndexBatch.New(indexActions); if (batch.Actions.Any()) { var startTime = DateTime.Now; indexClient.Documents.Index(batch); var endTime = DateTime.Now; var duration = queryEndTime - queryStartTime; page++; Console.WriteLine($"Processed {page * PageSize} learning aim documents"); Console.WriteLine($"query time: {duration.Minutes} mins, {duration.Seconds} secs, {duration.Milliseconds} ms)"); duration = postProcessEndTime - postProcessStartTime; Console.WriteLine($"post process time: {duration.Minutes} mins, {duration.Seconds} secs, {duration.Milliseconds} ms)"); duration = endTime - startTime; Console.WriteLine($"batch time: {duration.Minutes} mins, {duration.Seconds} secs, {duration.Milliseconds} ms) \n"); } else { next = false; } } } Console.WriteLine("Waiting for indexing...\n"); Thread.Sleep(2000); }
public async override Task PopulateIndexAsync(CancellationToken cancellationToken) { var indexClient = GetIndexClient(); LookUpModel lookups; using (var context = _contextFactory.GetLarsContext()) { lookups = new LookUpModel { LookUpKey = "1", NotionalNvqLevel2Lookups = await context.LarsNotionalNvqlevelv2Lookups .Select(lvl => new NotionalNVQLevel2LookupModel { NotionalNVQLevelV2 = lvl.NotionalNvqlevelV2, NotionalNVQLevelV2Desc = lvl.NotionalNvqlevelV2desc }).ToListAsync(cancellationToken), AcademicYearLookups = _academicYearService.GetAcademicYears(context) .Select(ay => new AcademicYearLookupModel { IsCurrentAcademicYear = _academicYearService.IsCurrentAcademicYear(ay), AcademicYear = ay.AcademicYear, AcademicYearDesc = _academicYearService.FormatDescription(ay.AcademicYearDesc) }).ToList(), AwardingBodyLookups = await context.LarsAwardOrgCodeLookups .Select(ab => new AwardingBodyLookupModel { AwardingBodyCode = ab.AwardOrgCode, AwardingBodyName = ab.AwardOrgName }).ToListAsync(cancellationToken), ValidityCategoryLookups = await context.LarsValidityCategoryLookups .Select(vc => new ValidityCategoryLookupModel { ValidityCategory = vc.ValidityCategory, ValidityCategoryDescription = vc.ValidityCategoryDesc2 }).ToListAsync(cancellationToken), ValidityFundingMappingLookups = await context.LarsValidityFundingMappings .Select(fm => new ValidityFundingMappingLookupModel { ValidityCategory = fm.ValidityCategory, FundingCategory = fm.FundingCategory, EffectiveFrom = fm.EffectiveFrom, EffectiveTo = fm.EffectiveTo }).ToListAsync(cancellationToken), FrameworkTypeLookups = await context.LarsProgTypeLookups .Where(ft => !_tlevelProgTypes.Contains(ft.ProgType)) .Select(ft => new FrameworkTypeLookupModel { FrameworkType = ft.ProgType.ToString(), FrameworkTypeDesc = ft.ProgTypeDesc }).ToListAsync(cancellationToken), TLevelTypeLookups = await context.LarsProgTypeLookups .Where(ft => _tlevelProgTypes.Contains(ft.ProgType)) .Select(ft => new FrameworkTypeLookupModel { FrameworkType = ft.ProgType.ToString(), FrameworkTypeDesc = ft.ProgTypeDesc }).ToListAsync(cancellationToken), IssuingAuthorityLookups = await context.LarsIssuingAuthorityLookups .Select(ia => new IssuingAuthorityLookupModel { IssuingAuthority = ia.IssuingAuthority.ToString(), IssuingAuthorityDesc = ia.IssuingAuthorityDesc }).ToListAsync(cancellationToken), StandardSectorLookups = await context.LarsStandardSectorCodeLookups .Select(sc => new StandardSectorLookupModel { StandardSectorCode = sc.StandardSectorCode, StandardSectorCodeDesc = sc.StandardSectorCodeDesc2 }).ToListAsync(cancellationToken), SectorSubjectAreaTier1Lookups = await context.LarsSectorSubjectAreaTier1Lookups .Select(st => new SectorSubjectAreaTier1LookupModel { SectorSubjectAreaTier1 = st.SectorSubjectAreaTier1.ToString(), SectorSubjectAreaTier1Desc = st.SectorSubjectAreaTier1Desc }).ToListAsync(cancellationToken) }; } Sort(lookups); var indexActions = new List <IndexAction <LookUpModel> > { IndexAction.Upload(lookups) }; var batch = IndexBatch.New(indexActions); if (batch.Actions.Any()) { indexClient.Documents.Index(batch); } }