private IEnumerable <Observation> GetObservations(
StatisticsDbContext context,
DataRowCollection rows,
List <string> headers,
Subject subject,
SubjectMeta subjectMeta,
int batchNo,
int rowsPerBatch
)
{
var observations = new List <Observation>();
var i = 0;
foreach (DataRow row in rows)
{
var o = ObservationFromCsv(
context,
CsvUtil.GetRowValues(row).ToArray(),
headers,
subject,
subjectMeta,
((batchNo - 1) * rowsPerBatch) + i++ + 2);
if (!IgnoredGeographicLevels.Contains(o.GeographicLevel))
{
observations.Add(o);
}
}
return(observations);
}
public async Task ImportFiltersAndLocations(DataColumnCollection cols, DataRowCollection rows,
SubjectMeta subjectMeta, StatisticsDbContext context, Guid releaseId, string dataFileName)
{
// Clearing the caches is required here as the seeder shares the cache with all subjects
_importerFilterService.ClearCache();
_importerLocationService.ClearCache();
var headers = CsvUtil.GetColumnValues(cols);
var rowCount = 1;
var totalRows = rows.Count;
foreach (DataRow row in rows)
{
if (rowCount % STAGE_2_ROW_CHECK == 0)
{
var currentStatus = await _importStatusService.GetImportStatus(releaseId, dataFileName);
if (currentStatus.IsFinishedOrAborting())
{
_logger.LogInformation($"Import for {dataFileName} has finished or is being aborted, " +
$"so finishing importing Filters and Locations early");
return;
}
await _importStatusService.UpdateStatus(releaseId,
dataFileName,
IStatus.STAGE_2,
(double)rowCount / totalRows * 100);
}
CreateFiltersAndLocationsFromCsv(context, CsvUtil.GetRowValues(row), headers, subjectMeta.Filters);
rowCount++;
}
}
public async Task ImportObservations(DataColumnCollection cols, DataRowCollection rows, Subject subject,
SubjectMeta subjectMeta, int batchNo, int rowsPerBatch, StatisticsDbContext context)
{
_importerFilterService.ClearCache();
_importerLocationService.ClearCache();
var observations = GetObservations(
context,
rows,
CsvUtil.GetColumnValues(cols),
subject,
subjectMeta,
batchNo,
rowsPerBatch).ToList();
await InsertObservations(context, observations);
}
private Observation ObservationFromCsv(
StatisticsDbContext context,
string[] line,
List <string> headers,
Subject subject,
SubjectMeta subjectMeta,
int csvRowNum)
{
var observationId = _guidGenerator.NewGuid();
return(new Observation
{
Id = observationId,
FilterItems = GetFilterItems(context, line, headers, subjectMeta.Filters, observationId),
GeographicLevel = GetGeographicLevel(line, headers),
LocationId = GetLocationId(line, headers, context),
Measures = GetMeasures(line, headers, subjectMeta.Indicators),
SubjectId = subject.Id,
TimeIdentifier = GetTimeIdentifier(line, headers),
Year = GetYear(line, headers),
CsvRow = csvRowNum
});
}
