public async Task Retrieve()
{
var cancellationToken = CancellationToken.None;
var fileServiceMock = new Mock <IFileService>();
var zipArchiveFileServiceMock = new Mock <IZipArchiveFileService>();
var inputReferenceDataContext = new Mock <IInputReferenceDataContext>();
var expectedReferenceData = new DesktopReferenceDataRoot
{
MetaDatas = TestNetaData(),
DevolvedPostcodes = new DevolvedPostcodes
{
McaGlaSofLookups = TestMcaSofLookups(),
Postcodes = TestDevolvedPostcodes(),
},
Employers = TestEmployers(),
EPAOrganisations = TestEpaOrgs(),
LARSLearningDeliveries = TestLarsLearningDeliveries(),
LARSFrameworkAims = TestLarsFrameworkAims(),
LARSFrameworks = TestLarsFrameworks(),
LARSStandards = TestLarsStandards(),
Organisations = TestOrganisations(),
Postcodes = TestPostcodes()
};
var currentPath = Directory.GetCurrentDirectory();
inputReferenceDataContext.Setup(r => r.InputReferenceDataFileKey).Returns("ReferenceData.zip");
inputReferenceDataContext.Setup(r => r.Container).Returns(currentPath);
zipArchiveFileServiceMock.Setup(zs => zs.RetrieveModel <MetaData>(It.IsAny <ZipArchive>(), It.IsAny <string>())).Returns(TestNetaData());
zipArchiveFileServiceMock.Setup(zs => zs.RetrieveModels <DevolvedPostcode>(It.IsAny <ZipArchive>(), It.IsAny <string>(), null)).Returns(TestDevolvedPostcodes());
zipArchiveFileServiceMock.Setup(zs => zs.RetrieveModels <McaGlaSofLookup>(It.IsAny <ZipArchive>(), It.IsAny <string>(), null)).Returns(TestMcaSofLookups());
zipArchiveFileServiceMock.Setup(zs => zs.RetrieveModels <McaGlaSofLookup>(It.IsAny <ZipArchive>(), It.IsAny <string>(), null)).Returns(TestMcaSofLookups());
zipArchiveFileServiceMock.Setup(zs => zs.RetrieveModels <Employer>(It.IsAny <ZipArchive>(), It.IsAny <string>(), null)).Returns(TestEmployers());
zipArchiveFileServiceMock.Setup(zs => zs.RetrieveModels <EPAOrganisation>(It.IsAny <ZipArchive>(), It.IsAny <string>(), null)).Returns(TestEpaOrgs());
zipArchiveFileServiceMock.Setup(zs => zs.RetrieveModels <LARSLearningDelivery>(It.IsAny <ZipArchive>(), It.IsAny <string>(), null)).Returns(TestLarsLearningDeliveries());
zipArchiveFileServiceMock.Setup(zs => zs.RetrieveModels <LARSFrameworkDesktop>(It.IsAny <ZipArchive>(), It.IsAny <string>(), null)).Returns(TestLarsFrameworks());
zipArchiveFileServiceMock.Setup(zs => zs.RetrieveModels <LARSFrameworkAimDesktop>(It.IsAny <ZipArchive>(), It.IsAny <string>(), null)).Returns(TestLarsFrameworkAims());
zipArchiveFileServiceMock.Setup(zs => zs.RetrieveModels <LARSStandard>(It.IsAny <ZipArchive>(), It.IsAny <string>(), null)).Returns(TestLarsStandards());
zipArchiveFileServiceMock.Setup(zs => zs.RetrieveModels <Organisation>(It.IsAny <ZipArchive>(), It.IsAny <string>(), null)).Returns(TestOrganisations());
zipArchiveFileServiceMock.Setup(zs => zs.RetrieveModels <Postcode>(It.IsAny <ZipArchive>(), It.IsAny <string>(), null)).Returns(TestPostcodes());
using (Stream stream = new FileStream(currentPath + "\\TestFiles\\ReferenceInputReferenceData.zip", FileMode.Open))
{
fileServiceMock.Setup(fs => fs.OpenReadStreamAsync(
inputReferenceDataContext.Object.InputReferenceDataFileKey,
inputReferenceDataContext.Object.Container,
cancellationToken)).ReturnsAsync(stream);
var result = await NewService(
zipArchiveFileServiceMock.Object,
fileServiceMock.Object)
.MapReferenceData(inputReferenceDataContext.Object, cancellationToken);
result.Should().BeEquivalentTo(expectedReferenceData);
}
}
public async Task ProcessAync()
{
var container = "Container";
var collectionName = "Collection";
var filePrefix = "FISReferenceData";
var versionNumber = "2";
var submissionDateTime = new DateTime(2020, 01, 01, 9, 00, 00);
var cancellationToken = CancellationToken.None;
var zipFileServiceMock = new Mock <IZipFileService>();
var loggerMock = new Mock <ILogger>();
var fileNameServiceMock = new Mock <IDesktopReferenceDataFileNameService>();
var desktopRefDataConfigMock = new Mock <IDesktopReferenceDataConfiguration>();
var context = new Mock <IDesktopReferenceDataContext>();
context.Setup(x => x.Container).Returns(container);
context.Setup(x => x.JobId).Returns(1);
context.Setup(x => x.CollectionName).Returns(collectionName);
context.Setup(x => x.SubmissionDateTimeUTC).Returns(submissionDateTime);
context.Setup(x => x.VersionNumber).Returns(versionNumber);
var filePath = $@"{context.Object.CollectionName}\{context.Object.JobId}";
var outputFileName = $@"{filePath}/FISReferenceData.2.zip";
fileNameServiceMock.Setup(fsm => fsm.BuildFileName(filePath, filePrefix, "2")).Returns(outputFileName);
var desktopReferenceDataRoot = new DesktopReferenceDataRoot();
zipFileServiceMock.Setup(s => s.SaveCollectionZipAsync(
outputFileName,
container,
It.IsAny <MetaData>(),
It.IsAny <DevolvedPostcodes>(),
It.IsAny <IReadOnlyCollection <Employer> >(),
It.IsAny <IReadOnlyCollection <EPAOrganisation> >(),
It.IsAny <IReadOnlyCollection <LARSFrameworkDesktop> >(),
It.IsAny <IReadOnlyCollection <LARSFrameworkAimDesktop> >(),
It.IsAny <IReadOnlyCollection <LARSLearningDelivery> >(),
It.IsAny <IReadOnlyCollection <LARSStandard> >(),
It.IsAny <IReadOnlyCollection <Organisation> >(),
It.IsAny <IReadOnlyCollection <Postcode> >(),
cancellationToken)).Returns(Task.CompletedTask).Verifiable();
desktopRefDataConfigMock.Setup(x => x.DesktopReferenceDataFilePreFix).Returns(filePrefix);
var service = NewService(desktopRefDataConfigMock.Object, fileNameServiceMock.Object, zipFileServiceMock.Object, loggerMock.Object);
await service.ProcessAsync(context.Object, desktopReferenceDataRoot, cancellationToken);
desktopRefDataConfigMock.VerifyAll();
fileNameServiceMock.VerifyAll();
zipFileServiceMock.VerifyAll();
}
private ReferenceDataRoot MapData(DesktopReferenceDataRoot desktopReferenceData, MapperData mapperData)
{
return(new ReferenceDataRoot
{
MetaDatas = desktopReferenceData.MetaDatas,
DevolvedPostcodes = desktopReferenceData.DevolvedPostcodes,
Employers = desktopReferenceData.Employers,
EPAOrganisations = desktopReferenceData.EPAOrganisations,
LARSLearningDeliveries = _larsLearningDeliveryMapperService.Map(mapperData.LARSLearningDeliveryKeys, desktopReferenceData),
LARSStandards = desktopReferenceData.LARSStandards,
Organisations = desktopReferenceData.Organisations,
Postcodes = desktopReferenceData.Postcodes,
});
}
public async Task <ReferenceDataRoot> MapReferenceData(IReferenceDataContext referenceDataContext, MapperData mapperData, CancellationToken cancellationToken)
{
var desktopReferenceData = new DesktopReferenceDataRoot();
using (var zipFileStream = await _fileService.OpenReadStreamAsync(referenceDataContext.DesktopInputReferenceDataFileKey, referenceDataContext.Container, cancellationToken))
{
using (var zip = new ZipArchive(zipFileStream, ZipArchiveMode.Read))
{
_logger.LogInfo("Reference Data - Retrieve MetaData");
desktopReferenceData.MetaDatas = _zipArchiveFileService.RetrieveModel <MetaData>(zip, DesktopReferenceDataConstants.MetaDataFile);
_logger.LogInfo("Reference Data - Retrieve Devolved Postcodes");
desktopReferenceData.DevolvedPostcodes = GetDevolvedPostcodes(zip, mapperData.Postcodes);
_logger.LogInfo("Reference Data - Retrieve Employers");
desktopReferenceData.Employers = _zipArchiveFileService.RetrieveModels <Employer>(zip, DesktopReferenceDataConstants.EmployersFile, x => mapperData.EmployerIds.Contains(x.ERN));
_logger.LogInfo("Reference Data - Retrieve Epa Organisations");
desktopReferenceData.EPAOrganisations = _zipArchiveFileService.RetrieveModels <EPAOrganisation>(zip, DesktopReferenceDataConstants.EPAOrganisationsFile, x => mapperData.EpaOrgIds.Contains(x.ID));
_logger.LogInfo("Reference Data - Retrieve Lars Frameworks");
desktopReferenceData.LARSFrameworks = _zipArchiveFileService.RetrieveModels <LARSFrameworkDesktop>(zip, DesktopReferenceDataConstants.LARSFrameworksFile);
_logger.LogInfo("Reference Data - Retrieve Lars Framework Aims");
desktopReferenceData.LARSFrameworkAims = _zipArchiveFileService.RetrieveModels <LARSFrameworkAimDesktop>(zip, DesktopReferenceDataConstants.LARSFrameworkAimsFile);
_logger.LogInfo("Reference Data - Retrieve Lars Learning Deliveries");
desktopReferenceData.LARSLearningDeliveries = _zipArchiveFileService.RetrieveModels <LARSLearningDelivery>(zip, DesktopReferenceDataConstants.LARSLearningDeliveriesFile, x => mapperData.LARSLearningDeliveryKeys.Select(l => l.LearnAimRef).Contains(x.LearnAimRef, StringComparer.OrdinalIgnoreCase));
_logger.LogInfo("Reference Data - Retrieve Lars Standards");
desktopReferenceData.LARSStandards = _zipArchiveFileService.RetrieveModels <LARSStandard>(zip, DesktopReferenceDataConstants.LARSStandardsFile, x => mapperData.StandardCodes.Contains(x.StandardCode));
_logger.LogInfo("Reference Data - Retrieve Organisations");
desktopReferenceData.Organisations = _zipArchiveFileService.RetrieveModels <Organisation>(zip, DesktopReferenceDataConstants.OrganisationsFile, x => mapperData.UKPRNs.Contains(x.UKPRN));
_logger.LogInfo("Reference Data - Retrieve Postcodes");
desktopReferenceData.Postcodes = _zipArchiveFileService.RetrieveModels <Postcode>(zip, DesktopReferenceDataConstants.PostcodesFile, x => mapperData.Postcodes.Contains(x.PostCode));
}
}
var referenceDataRoot = MapData(desktopReferenceData, mapperData);
return(referenceDataRoot);
}
public async Task <DesktopReferenceDataRoot> MapReferenceData(IInputReferenceDataContext inputReferenceDataContext, CancellationToken cancellationToken)
{
var desktopReferenceData = new DesktopReferenceDataRoot();
using (var zipFileStream = await _fileService.OpenReadStreamAsync(inputReferenceDataContext.InputReferenceDataFileKey, inputReferenceDataContext.Container, cancellationToken))
{
using (var zip = new ZipArchive(zipFileStream, ZipArchiveMode.Read))
{
_logger.LogInfo("Reference Data - Retrieve MetaData");
desktopReferenceData.MetaDatas = _zipArchiveFileService.RetrieveModel <MetaData>(zip, DesktopReferenceDataConstants.MetaDataFile);
_logger.LogInfo("Reference Data - Retrieve Devolved Postcodes");
desktopReferenceData.DevolvedPostcodes = GetDevolvedPostcodes(zip);
_logger.LogInfo("Reference Data - Retrieve Employers");
desktopReferenceData.Employers = _zipArchiveFileService.RetrieveModels <Employer>(zip, DesktopReferenceDataConstants.EmployersFile);
_logger.LogInfo("Reference Data - Retrieve Epa Organisations");
desktopReferenceData.EPAOrganisations = _zipArchiveFileService.RetrieveModels <EPAOrganisation>(zip, DesktopReferenceDataConstants.EPAOrganisationsFile);
_logger.LogInfo("Reference Data - Retrieve Lars Frameworks");
desktopReferenceData.LARSFrameworks = _zipArchiveFileService.RetrieveModels <LARSFrameworkDesktop>(zip, DesktopReferenceDataConstants.LARSFrameworksFile);
_logger.LogInfo("Reference Data - Retrieve Lars Framework Aims");
desktopReferenceData.LARSFrameworkAims = _zipArchiveFileService.RetrieveModels <LARSFrameworkAimDesktop>(zip, DesktopReferenceDataConstants.LARSFrameworkAimsFile);
_logger.LogInfo("Reference Data - Retrieve Lars Learning Deliveries");
desktopReferenceData.LARSLearningDeliveries = _zipArchiveFileService.RetrieveModels <LARSLearningDelivery>(zip, DesktopReferenceDataConstants.LARSLearningDeliveriesFile);
_logger.LogInfo("Reference Data - Retrieve Lars Standards");
desktopReferenceData.LARSStandards = _zipArchiveFileService.RetrieveModels <LARSStandard>(zip, DesktopReferenceDataConstants.LARSStandardsFile);
_logger.LogInfo("Reference Data - Retrieve Organisations");
desktopReferenceData.Organisations = _zipArchiveFileService.RetrieveModels <Organisation>(zip, DesktopReferenceDataConstants.OrganisationsFile);
_logger.LogInfo("Reference Data - Retrieve Postcodes");
desktopReferenceData.Postcodes = _zipArchiveFileService.RetrieveModels <Postcode>(zip, DesktopReferenceDataConstants.PostcodesFile);
}
}
return(desktopReferenceData);
}
public async Task ProcessAsync(IDesktopReferenceDataContext context, DesktopReferenceDataRoot desktopReferenceDataRoot, CancellationToken cancellationToken)
{
_logger.LogInfo("Generating Desktop Reference Data File.");
var filePath = BuildFilePath(context);
var fileName = _desktopReferenceDataFileNameService.BuildFileName(filePath, _desktopRefDataConfig.DesktopReferenceDataFilePreFix, context.VersionNumber);
await _zipFileService.SaveCollectionZipAsync(
fileName,
context.Container,
desktopReferenceDataRoot.MetaDatas,
desktopReferenceDataRoot.DevolvedPostcodes,
desktopReferenceDataRoot.Employers,
desktopReferenceDataRoot.EPAOrganisations,
desktopReferenceDataRoot.LARSFrameworks,
desktopReferenceDataRoot.LARSFrameworkAims,
desktopReferenceDataRoot.LARSLearningDeliveries,
desktopReferenceDataRoot.LARSStandards,
desktopReferenceDataRoot.Organisations,
desktopReferenceDataRoot.Postcodes,
cancellationToken);
}
public async Task Retrieve()
{
var cancellationToken = CancellationToken.None;
var referenceDataContextMock = new Mock <IDesktopReferenceDataContext>();
var referenceDataPopulationServiceMock = new Mock <IDesktopReferenceDataPopulationService>();
var desktopReferenceDataFileServiceMock = new Mock <IDesktopReferenceDataFileService>();
var desktopReferenceDataSummaryFileServiceMock = new Mock <IDesktopReferenceDataSummaryFileService>();
var loggerMock = new Mock <ILogger>();
var desktopReferenceDataRoot = new DesktopReferenceDataRoot();
referenceDataPopulationServiceMock.Setup(s => s.PopulateAsync(cancellationToken)).Returns(Task.FromResult(desktopReferenceDataRoot)).Verifiable();
desktopReferenceDataFileServiceMock.Setup(s => s.ProcessAsync(referenceDataContextMock.Object, desktopReferenceDataRoot, cancellationToken)).Returns(Task.CompletedTask).Verifiable();
desktopReferenceDataSummaryFileServiceMock.Setup(s => s.ProcessAync(referenceDataContextMock.Object, cancellationToken)).Returns(Task.CompletedTask).Verifiable();
var service = NewService(referenceDataPopulationServiceMock.Object, desktopReferenceDataFileServiceMock.Object, desktopReferenceDataSummaryFileServiceMock.Object, loggerMock.Object);
await service.ExecuteAsync(referenceDataContextMock.Object, cancellationToken);
referenceDataPopulationServiceMock.VerifyAll();
desktopReferenceDataFileServiceMock.VerifyAll();
}
public IReadOnlyCollection <LARSLearningDelivery> Map(IReadOnlyCollection <LARSLearningDeliveryKey> input, DesktopReferenceDataRoot referenceData)
{
var larsFrameworks = new List <LARSFrameworkKey>();
var larsLearningDeliveries = referenceData.LARSLearningDeliveries
.Where(l => input.Select(lldk => lldk.LearnAimRef).Contains(l.LearnAimRef, StringComparer.OrdinalIgnoreCase))
.ToList();
var larsFrameworkAimsDictionary = referenceData.LARSFrameworkAims
.GroupBy(ld => new LARSLearningDeliveryKey(ld.LearnAimRef, ld.FworkCode, ld.ProgType, ld.PwayCode))
.ToDictionary(
k => k.Key,
v => v.Select(fa => new LARSFrameworkAim
{
LearnAimRef = fa.LearnAimRef,
FrameworkComponentType = fa.FrameworkComponentType,
EffectiveFrom = fa.EffectiveFrom,
EffectiveTo = fa.EffectiveTo,
}).FirstOrDefault());
foreach (var key in input)
{
var framework = referenceData.LARSFrameworks
.Where(lf =>
lf.FworkCode == key.FworkCode &&
lf.ProgType == key.ProgType &&
lf.PwayCode == key.PwayCode)
.Select(l => new LARSFramework
{
FworkCode = l.FworkCode,
PwayCode = l.PwayCode,
ProgType = l.ProgType,
EffectiveFromNullable = l.EffectiveFromNullable,
EffectiveTo = l.EffectiveTo,
LARSFrameworkApprenticeshipFundings = l.LARSFrameworkApprenticeshipFundings,
LARSFrameworkCommonComponents = l.LARSFrameworkCommonComponents,
LARSFrameworkAim = larsFrameworkAimsDictionary.TryGetValue(key, out var frameworkAims) ? frameworkAims : null,
}).FirstOrDefault();
