private IEnumerable <Observation> GetObservations( StatisticsDbContext context, DataRowCollection rows, List <string> headers, Subject subject, SubjectMeta subjectMeta, int batchNo, int rowsPerBatch ) { var observations = new List <Observation>(); var i = 0; foreach (DataRow row in rows) { var o = ObservationFromCsv( context, CsvUtil.GetRowValues(row).ToArray(), headers, subject, subjectMeta, ((batchNo - 1) * rowsPerBatch) + i++ + 2); if (!IgnoredGeographicLevels.Contains(o.GeographicLevel)) { observations.Add(o); } } return(observations); }
public async Task ImportFiltersAndLocations(DataColumnCollection cols, DataRowCollection rows, SubjectMeta subjectMeta, StatisticsDbContext context, Guid releaseId, string dataFileName) { // Clearing the caches is required here as the seeder shares the cache with all subjects _importerFilterService.ClearCache(); _importerLocationService.ClearCache(); var headers = CsvUtil.GetColumnValues(cols); var rowCount = 1; var totalRows = rows.Count; foreach (DataRow row in rows) { if (rowCount % STAGE_2_ROW_CHECK == 0) { var currentStatus = await _importStatusService.GetImportStatus(releaseId, dataFileName); if (currentStatus.IsFinishedOrAborting()) { _logger.LogInformation($"Import for {dataFileName} has finished or is being aborted, " + $"so finishing importing Filters and Locations early"); return; } await _importStatusService.UpdateStatus(releaseId, dataFileName, IStatus.STAGE_2, (double)rowCount / totalRows * 100); } CreateFiltersAndLocationsFromCsv(context, CsvUtil.GetRowValues(row), headers, subjectMeta.Filters); rowCount++; } }
public async Task ImportObservations(DataColumnCollection cols, DataRowCollection rows, Subject subject, SubjectMeta subjectMeta, int batchNo, int rowsPerBatch, StatisticsDbContext context) { _importerFilterService.ClearCache(); _importerLocationService.ClearCache(); var observations = GetObservations( context, rows, CsvUtil.GetColumnValues(cols), subject, subjectMeta, batchNo, rowsPerBatch).ToList(); await InsertObservations(context, observations); }
private Observation ObservationFromCsv( StatisticsDbContext context, string[] line, List <string> headers, Subject subject, SubjectMeta subjectMeta, int csvRowNum) { var observationId = _guidGenerator.NewGuid(); return(new Observation { Id = observationId, FilterItems = GetFilterItems(context, line, headers, subjectMeta.Filters, observationId), GeographicLevel = GetGeographicLevel(line, headers), LocationId = GetLocationId(line, headers, context), Measures = GetMeasures(line, headers, subjectMeta.Indicators), SubjectId = subject.Id, TimeIdentifier = GetTimeIdentifier(line, headers), Year = GetYear(line, headers), CsvRow = csvRowNum }); }