public async Task <IEnumerable <DownloadDetailsModel> > GetDownloadDetails(CancellationToken cancellationToken)
{
var result = new List <DownloadDetailsModel>();
List <LarsVersion> versions;
using (var context = _contextFactory.GetLarsContext())
{
versions = await context.LarsVersions
.OrderByDescending(v => v.MajorNumber)
.Take(2)
.ToListAsync(cancellationToken);
}
var id = 0;
foreach (var version in versions)
{
result.Add(CreateDownloadDetailsModel(version, CSVTypeName, ref id));
result.Add(CreateDownloadDetailsModel(version, MDBTypeName, ref id));
result.Add(CreateDownloadDetailsModel(version, PSVTypeName, ref id));
}
return(result);
}
public async override Task PopulateIndexAsync(CancellationToken cancellationToken)
{
var indexClient = GetIndexClient();
IEnumerable <StandardModel> standards;
using (var context = _contextFactory.GetLarsContext())
{
var standardSectorCodes = await _standardSectorCodeService.GetStandardSectorCodeDescriptionsAsync(context);
var commonComponents = await _commonComponentService.GetStandardCommonComponents(context);
var relatedLearningAims = await _relatedLearningAimsService.GetStandardRelatedLearningAims(context);
var commonComponentLookups = await _commonComponentLookupService.GetCommonComponentLookupsAsync(context);
var componentTypes = await _componentTypeService.GetComponentTypesAsync(context);
standards = await context.LarsStandards
.Select(st => new StandardModel
{
StandardCode = st.StandardCode.ToString(),
StandardName = st.StandardName,
StandardSectorCode = st.StandardSectorCode,
StandardSectorCodeDesc2 = standardSectorCodes[st.StandardSectorCode],
Version = st.Version,
NotionalEndLevel = st.NotionalEndLevel,
EffectiveFrom = st.EffectiveFrom,
LastDateStarts = st.LastDateStarts,
EffectiveTo = st.EffectiveTo,
SectorSubjectAreaTier1 = st.SectorSubjectAreaTier1.ToString(),
SectorSubjectAreaTier1Desc = st.SectorSubjectAreaTier1Navigation.SectorSubjectAreaTier1Desc,
SectorSubjectAreaTier2 = st.SectorSubjectAreaTier2.ToString(),
SectorSubjectAreaTier2Desc = st.SectorSubjectAreaTier2Navigation.SectorSubjectAreaTier2Desc,
IntegratedDegreeStandard = st.IntegratedDegreeStandard,
OtherBodyApprovalRequired = st.OtherBodyApprovalRequired,
StandardFundingModels = st.LarsStandardFundings
.Select(sf => new StandardFundingModel
{
FundingCategoryDescription = sf.FundingCategoryNavigation.FundingCategoryDesc2,
BandNumber = sf.BandNumber,
EffectiveFrom = sf.EffectiveFrom,
EffectiveTo = sf.EffectiveTo,
CoreGovContributionCap = sf.CoreGovContributionCap.ToString(),
Incentive1618 = sf._1618incentive.ToString(),
SmallBusinessIncentive = sf.SmallBusinessIncentive.ToString(),
AchievementIncentive = sf.AchievementIncentive.ToString()
}).ToList(),
StandardApprenticeshipFundingModels = st.LarsApprenticeshipStdFundings
.Select(saf => new StandardApprenticeshipFundingModel
{
FundingCategoryDescription = saf.FundingCategoryNavigation.FundingCategoryDesc2,
BandNumber = saf.BandNumber,
EffectiveFrom = saf.EffectiveFrom,
EffectiveTo = saf.EffectiveTo,
ProviderAdditionalPayment1618 = saf._1618providerAdditionalPayment.ToString(),
EmployerAdditionalPayment1618 = saf._1618employerAdditionalPayment.ToString(),
FrameworkUplift1618 = saf._1618frameworkUplift.ToString(),
CareLeaverAdditionalPayment = saf.CareLeaverAdditionalPayment.ToString(),
Duration = saf.Duration.ToString(),
MaxEmployerLevyCap = saf.MaxEmployerLevyCap.ToString(),
FundableWithoutEmployer = saf.FundableWithoutEmployer
}).ToList()
}).ToListAsync(cancellationToken);
foreach (var standard in standards)
{
standard.CommonComponents = commonComponents[standard.StandardCode].ToList();
standard.CommonComponents.ForEach(c => c.Description = commonComponentLookups.GetValueOrDefault(c.CommonComponent)?.Description);
standard.LearningAims = relatedLearningAims[standard.StandardCode].ToList();
standard.LearningAims.ForEach(l => l.ComponentTypeDesc = componentTypes.GetValueOrDefault(l.ComponentType.Value));
}
}
var indexActions = standards.Select(IndexAction.Upload);
var batch = IndexBatch.New(indexActions);
if (batch.Actions.Any())
{
indexClient.Documents.Index(batch);
}
}
public async Task PopulateIndexAsync(bool units)
{
var indexClient = GetIndexClient();
using (var context = _contextFactory.GetLarsContext())
{
// For an unknown reason large datasets with complex joins appear to randomly
// drop data. Not sure why but advised approach is to preload data and query
// that rather than do all in single EF query.
var academicYears = _academicYearService.GetAcademicYears(context);
var issuingAuthorities = await _issuingAuthorityService.GetIssuingAuthoritiesAsync(context);
var componentTypes = await _componentTypeService.GetComponentTypesAsync(context);
var fundings = await _fundingService.GetFundingsAsync(context);
var categories = await _learningDeliveryCategoryService.GetLearningDeliveryCategoriesAsync(context);
var validities = await _validityService.GetValiditiesAsync(context);
var entitlementCategories = await _entitlementCategoryService.GetEntitlementCategoriesAsync(context);
var frameworkAims = await _frameworkAimService.GetLearningAimFrameworkAimsAsync(context, units);
var awardBodyCodes = await _awardOrgService.GetAwardingOrgNamesAsync(context);
var page = 0;
var next = true;
while (next)
{
var queryStartTime = DateTime.Now;
var learningAimsQueryable = context.LarsLearningDeliveries.AsQueryable();
if (units)
{
learningAimsQueryable = learningAimsQueryable.Where(ld => ld.LearnAimRefType == UnitLearnAimRefType);
}
else
{
learningAimsQueryable = learningAimsQueryable.Where(ld => ld.LearnAimRefType != UnitLearnAimRefType);
}
var learningAims = learningAimsQueryable
.OrderBy(ld => ld.LearnAimRef)
.ThenBy(ld => ld.EffectiveFrom)
.Skip(page * PageSize)
.Take(PageSize)
.Select(ld => new LearningAimModel
{
LearnAimRef = ld.LearnAimRef,
AwardingBodyCode = ld.AwardOrgCode,
EffectiveFrom = ld.EffectiveFrom,
EffectiveTo = ld.EffectiveTo,
Level = ld.NotionalNvqlevelv2Navigation.NotionalNvqlevelV2,
LevelDescription = ld.NotionalNvqlevelv2Navigation.NotionalNvqlevelV2desc,
Type = ld.LearnAimRefTypeNavigation.LearnAimRefTypeDesc,
LearningAimTitle = ld.LearnAimRefTitle,
GuidedLearningHours = ld.GuidedLearningHours.ToString(),
IsOFQUAL = ld.LearningDeliveryGenre == OFQUALRegulatedQualification ||
ld.LearningDeliveryGenre == OFQUALRegulatedUnit
})
.ToArray();
var queryEndTime = DateTime.Now;
var postProcessStartTime = DateTime.Now;
foreach (var learningDelivery in learningAims)
{
PopulateFrameworks(learningDelivery, frameworkAims, issuingAuthorities, componentTypes);
PopulateCategories(learningDelivery, categories.GetValueOrDefault(learningDelivery.LearnAimRef, new List <CategoryModel>()));
learningDelivery.AwardingBodyName = awardBodyCodes.GetValueOrDefault(learningDelivery.AwardingBodyCode);
learningDelivery.GuidedLearningHours = GetGuidedLearningHours(learningDelivery.GuidedLearningHours);
var fundingForDelivery = fundings.GetValueOrDefault(learningDelivery.LearnAimRef, new List <FundingModel>());
var validityForDelivery = validities.GetValueOrDefault(learningDelivery.LearnAimRef, new List <ValidityModel>());
var entitlementCategory = entitlementCategories.GetValueOrDefault(learningDelivery.LearnAimRef, new List <EntitlementCategoryModel>());
PopulateAcademicYears(learningDelivery, academicYears.ToList(), fundingForDelivery, entitlementCategory, validityForDelivery);
}
var postProcessEndTime = DateTime.Now;
var indexActions = learningAims.Select(IndexAction.Upload);
var batch = IndexBatch.New(indexActions);
if (batch.Actions.Any())
{
var startTime = DateTime.Now;
indexClient.Documents.Index(batch);
var endTime = DateTime.Now;
var duration = queryEndTime - queryStartTime;
page++;
Console.WriteLine($"Processed {page * PageSize} learning aim documents");
Console.WriteLine($"query time: {duration.Minutes} mins, {duration.Seconds} secs, {duration.Milliseconds} ms)");
duration = postProcessEndTime - postProcessStartTime;
Console.WriteLine($"post process time: {duration.Minutes} mins, {duration.Seconds} secs, {duration.Milliseconds} ms)");
duration = endTime - startTime;
Console.WriteLine($"batch time: {duration.Minutes} mins, {duration.Seconds} secs, {duration.Milliseconds} ms) \n");
}
else
{
next = false;
}
}
}
Console.WriteLine("Waiting for indexing...\n");
Thread.Sleep(2000);
}
public async override Task PopulateIndexAsync(CancellationToken cancellationToken)
{
var indexClient = GetIndexClient();
LookUpModel lookups;
using (var context = _contextFactory.GetLarsContext())
{
lookups = new LookUpModel
{
LookUpKey = "1",
NotionalNvqLevel2Lookups = await context.LarsNotionalNvqlevelv2Lookups
.Select(lvl => new NotionalNVQLevel2LookupModel
{
NotionalNVQLevelV2 = lvl.NotionalNvqlevelV2,
NotionalNVQLevelV2Desc = lvl.NotionalNvqlevelV2desc
}).ToListAsync(cancellationToken),
AcademicYearLookups = _academicYearService.GetAcademicYears(context)
.Select(ay => new AcademicYearLookupModel
{
IsCurrentAcademicYear = _academicYearService.IsCurrentAcademicYear(ay),
AcademicYear = ay.AcademicYear,
AcademicYearDesc = _academicYearService.FormatDescription(ay.AcademicYearDesc)
}).ToList(),
AwardingBodyLookups = await context.LarsAwardOrgCodeLookups
.Select(ab => new AwardingBodyLookupModel
{
AwardingBodyCode = ab.AwardOrgCode,
AwardingBodyName = ab.AwardOrgName
}).ToListAsync(cancellationToken),
ValidityCategoryLookups = await context.LarsValidityCategoryLookups
.Select(vc => new ValidityCategoryLookupModel
{
ValidityCategory = vc.ValidityCategory,
ValidityCategoryDescription = vc.ValidityCategoryDesc2
}).ToListAsync(cancellationToken),
ValidityFundingMappingLookups = await context.LarsValidityFundingMappings
.Select(fm => new ValidityFundingMappingLookupModel
{
ValidityCategory = fm.ValidityCategory,
FundingCategory = fm.FundingCategory,
EffectiveFrom = fm.EffectiveFrom,
EffectiveTo = fm.EffectiveTo
}).ToListAsync(cancellationToken),
FrameworkTypeLookups = await context.LarsProgTypeLookups
.Where(ft => !_tlevelProgTypes.Contains(ft.ProgType))
.Select(ft => new FrameworkTypeLookupModel
{
FrameworkType = ft.ProgType.ToString(),
FrameworkTypeDesc = ft.ProgTypeDesc
}).ToListAsync(cancellationToken),
TLevelTypeLookups = await context.LarsProgTypeLookups
.Where(ft => _tlevelProgTypes.Contains(ft.ProgType))
.Select(ft => new FrameworkTypeLookupModel
{
FrameworkType = ft.ProgType.ToString(),
FrameworkTypeDesc = ft.ProgTypeDesc
}).ToListAsync(cancellationToken),
IssuingAuthorityLookups = await context.LarsIssuingAuthorityLookups
.Select(ia => new IssuingAuthorityLookupModel
{
IssuingAuthority = ia.IssuingAuthority.ToString(),
IssuingAuthorityDesc = ia.IssuingAuthorityDesc
}).ToListAsync(cancellationToken),
StandardSectorLookups = await context.LarsStandardSectorCodeLookups
.Select(sc => new StandardSectorLookupModel
{
StandardSectorCode = sc.StandardSectorCode,
StandardSectorCodeDesc = sc.StandardSectorCodeDesc2
}).ToListAsync(cancellationToken),
SectorSubjectAreaTier1Lookups = await context.LarsSectorSubjectAreaTier1Lookups
.Select(st => new SectorSubjectAreaTier1LookupModel
{
SectorSubjectAreaTier1 = st.SectorSubjectAreaTier1.ToString(),
SectorSubjectAreaTier1Desc = st.SectorSubjectAreaTier1Desc
}).ToListAsync(cancellationToken)
};
}
Sort(lookups);
var indexActions = new List <IndexAction <LookUpModel> > {
IndexAction.Upload(lookups)
};
var batch = IndexBatch.New(indexActions);
if (batch.Actions.Any())
{
indexClient.Documents.Index(batch);
}
}
public async Task PopulateIndexAsync(bool isFramework)
{
var indexClient = GetIndexClient();
IEnumerable <FrameworkModel> frameworks;
using (var context = _contextFactory.GetLarsContext())
{
var issuingAuthorities = await _issuingAuthorityService.GetIssuingAuthoritiesAsync(context);
var componentTypes = await _componentTypeService.GetComponentTypesAsync(context);
var commonComponents = await _commonComponentService.GetFrameworkCommonComponents(context);
var commonComponentLookups = await _commonComponentLookupService.GetCommonComponentLookupsAsync(context);
var relatedLearningAims = await _relatedLearningAimsService.GetFrameworkRelatedLearningAims(context);
var frameworksQueryable = context.LarsFrameworks.AsQueryable();
if (isFramework)
{
frameworksQueryable = frameworksQueryable.Where(f => !_tlevelProgTypes.Contains(f.ProgType));
}
else
{
frameworksQueryable = frameworksQueryable.Where(f => _tlevelProgTypes.Contains(f.ProgType));
}
frameworks = await frameworksQueryable
.Select(fr => new FrameworkModel
{
// azure search index must have 1 key field. Please ensure pattern here is the same as used in common components
// and releated learning aim lookups.
Id = string.Concat(fr.FworkCode, "-", fr.ProgType, "-", fr.PwayCode),
FrameworkCode = fr.FworkCode,
ProgramType = fr.ProgType,
PathwayCode = fr.PwayCode,
PathwayName = fr.PathwayName,
ProgramTypeName = fr.ProgTypeNavigation.ProgTypeDesc,
SearchableFrameworkCode = fr.FworkCode.ToString(),
FrameworkTitle = fr.IssuingAuthorityTitle,
EffectiveFrom = fr.EffectiveFrom,
EffectiveTo = fr.EffectiveTo,
SectorSubjectAreaTier1 = fr.SectorSubjectAreaTier1.ToString(), // decimal not supported by azure
SectorSubjectAreaTier1Desc = fr.SectorSubjectAreaTier1Navigation.SectorSubjectAreaTier1Desc,
SectorSubjectAreaTier2 = fr.SectorSubjectAreaTier2.ToString(), // decimal not supported by azure
SectorSubjectAreaTier2Desc = fr.SectorSubjectAreaTier2Navigation.SectorSubjectAreaTier2Desc,
IssuingAuthority = fr.IssuingAuthority,
IssuingAuthorityDesc = issuingAuthorities[fr.IssuingAuthority],
IsTLevel = !isFramework,
NasTitle = fr.Nastitle
}).ToListAsync();
foreach (var framework in frameworks)
{
framework.CommonComponents = commonComponents[framework.Id].ToList();
framework.CommonComponents.ForEach(c => c.Description = commonComponentLookups.GetValueOrDefault(c.CommonComponent)?.Description);
framework.LearningAims = relatedLearningAims[framework.Id].ToList();
framework.LearningAims.ForEach(l => l.ComponentTypeDesc = componentTypes.GetValueOrDefault(l.ComponentType.Value));
}
}
var indexActions = frameworks.Select(IndexAction.Upload);
var batch = IndexBatch.New(indexActions);
if (batch.Actions.Any())
{
indexClient.Documents.Index(batch);
}
}
