public async Task ImportDataIntoStaging()
{
try
{
_logger.LogInformation("Standards import - starting");
var standards = (await _instituteOfApprenticeshipService.GetStandards()).ToList();
_logger.LogInformation($"Standards import - Retrieved {standards.Count} standards from API");
var routes = GetDistinctRoutesFromStandards(standards);
await LoadRoutesInStaging(routes);
UpdateStandardsWithRespectiveSectorId(standards, routes);
var standardsImport = standards
.Select(c => (StandardImport)c)
.ToList();
_standardImportRepository.DeleteAll();
await _standardImportRepository.InsertMany(standardsImport);
_logger.LogInformation("Standards import - starting");
}
catch (Exception e)
{
_logger.LogError("Standards import - an error occurred when trying to import data into staging.", e);
throw;
}
}
public async Task ImportDataIntoStaging()
{
try
{
_logger.LogInformation("Standards import - starting");
var standards = (await _instituteOfApprenticeshipService.GetStandards()).ToList();
_logger.LogInformation($"Standards import - Retrieved {standards.Count} standards from API");
var routes = GetDistinctRoutesFromStandards(standards);
await LoadRoutesInStaging(routes);
UpdateStandardsWithRespectiveSectorId(standards, routes);
var standardsImport = standards
.Select(c => (StandardImport)c)
.ToList();
_standardImportRepository.DeleteAll();
var duplicates = standardsImport.GroupBy(s => s.StandardUId)
.Where(g => g.Count() > 1)
.Select(t => new { StandardUId = t.Key, Standards = t.ToList() });
foreach (var duplicate in duplicates)
{
var latestStandard = duplicate.Standards.OrderByDescending(d => d.CreatedDate.GetValueOrDefault()).FirstOrDefault();
standardsImport.RemoveAll(s => duplicate.StandardUId == s.StandardUId);
standardsImport.Add(latestStandard);
}
await _standardImportRepository.InsertMany(standardsImport);
_logger.LogInformation("Standards import - starting");
}
catch (Exception e)
{
_logger.LogError("Standards import - an error occurred when trying to import data into staging.", e);
throw;
}
}