private async Task InsertDataFromZipStreamToImportTables(Stream content)
{
_logger.LogInformation("LARS Import - starting extract from ZIP");
var standardsCsv = _zipArchiveHelper
.ExtractModelFromCsvFileZipStream <StandardCsv>(content, Constants.LarsStandardsFileName);
var apprenticeshipFundingCsv = _zipArchiveHelper
.ExtractModelFromCsvFileZipStream <ApprenticeshipFundingCsv>(content,
Constants.LarsApprenticeshipFundingFileName);
var sectorSubjectAreaTier2Csv = _zipArchiveHelper
.ExtractModelFromCsvFileZipStream <SectorSubjectAreaTier2Csv>(content,
Constants.LarsSectorSubjectAreaTier2FileName);
ClearStagingTables();
var larsImportResult = _larsStandardImportRepository
.InsertMany(standardsCsv.Select(c => (LarsStandardImport)c).ToList());
var filterRecords = apprenticeshipFundingCsv
.Where(c => c.ApprenticeshipType.StartsWith("STD", StringComparison.CurrentCultureIgnoreCase))
.Select(c => (ApprenticeshipFundingImport)c).ToList();
var apprenticeFundingImportResult =
_apprenticeshipFundingImportRepository.InsertMany(filterRecords);
var sectorSubjectAreaTier2ImportResult =
_sectorSubjectAreaTier2ImportRepository.InsertMany(sectorSubjectAreaTier2Csv
.Select(x => (SectorSubjectAreaTier2Import)x).ToList());
await Task.WhenAll(larsImportResult, apprenticeFundingImportResult, sectorSubjectAreaTier2ImportResult);
_logger.LogInformation("LARS Import - finished load into Import tables");
}
public async Task ImportData()
{
var timeStarted = DateTime.UtcNow;
var downloadFilePathTask = _pageParser.GetCurrentDownloadFilePath();
var auditTask = _auditRepository.GetLastImportByType(ImportType.NationalAchievementRatesOverall);
await Task.WhenAll(downloadFilePathTask, auditTask);
if (auditTask.Result != null && auditTask.Result.FileName.Equals(downloadFilePathTask.Result,
StringComparison.CurrentCultureIgnoreCase))
{
_logger.LogInformation("No new overall achievement rate data to load");
return;
}
var dataFile = await _downloadService.GetFileStream(downloadFilePathTask.Result);
var data = _zipArchiveHelper.ExtractModelFromCsvFileZipStream <NationalAchievementRateOverallCsv>(dataFile, Constants.NationalAchievementRatesOverallCsvFileName);
_logger.LogInformation("Clearing import table");
_importRepository.DeleteAll();
_logger.LogInformation("Loading to import table");
await _importRepository.InsertMany(data
.Where(c =>
!c.SectorSubjectArea.Contains("All Sector Subject Area"))
.Where(c => c.InstitutionType == "All Institution Type")
.Where(c => c.Age == "All Age")
.Select(c => (NationalAchievementRateOverallImport)c)
.ToList());
_logger.LogInformation("Clearing main table");
_repository.DeleteAll();
_logger.LogInformation("Importing to main table");
var items = (await _importRepository.GetAllWithAchievementData()).ToList();
await _repository.InsertMany(items.Select(c => (NationalAchievementRateOverall)c).ToList());
await _auditRepository.Insert(new ImportAudit(timeStarted, items.Count, ImportType.NationalAchievementRatesOverall,
downloadFilePathTask.Result));
_logger.LogInformation("Import Complete");
}