private static async Task <Either <IEnumerable <ValidationError>, Unit> > ValidateObservationHeaders(DataColumnCollection cols)
{
var errors = new List <ValidationError>();
foreach (var mandatoryCol in MandatoryObservationColumns)
{
if (!cols.Contains(mandatoryCol))
{
errors.Add(new ValidationError($"{DataFileMissingExpectedColumn.GetEnumLabel()} : {mandatoryCol}"));
}
if (errors.Count == 100)
{
errors.Add(new ValidationError(FirstOneHundredErrors.GetEnumLabel()));
break;
}
}
if (errors.Count > 0)
{
return(errors);
}
return(Unit.Instance);
}
ValidateAndCountObservations(
DataColumnCollection cols,
DataRowCollection rows,
ExecutionContext executionContext,
Guid releaseId,
string dataFileName)
{
var idx = 0;
var filteredRows = 0;
var totalRowCount = 0;
var errors = new List <ValidationError>();
var dataRows = rows.Count;
foreach (DataRow row in rows)
{
idx++;
if (errors.Count == 100)
{
errors.Add(new ValidationError(FirstOneHundredErrors.GetEnumLabel()));
break;
}
try
{
var rowValues = CsvUtil.GetRowValues(row);
var colValues = CsvUtil.GetColumnValues(cols);
ImporterService.GetGeographicLevel(rowValues, colValues);
ImporterService.GetTimeIdentifier(rowValues, colValues);
ImporterService.GetYear(rowValues, colValues);
if (!IsGeographicLevelIgnored(rowValues, colValues))
{
filteredRows++;
}
}
catch (Exception e)
{
errors.Add(new ValidationError($"error at row {idx}: {e.Message}"));
}
totalRowCount++;
if (totalRowCount % STAGE_1_ROW_CHECK == 0)
{
await _importStatusService.UpdateStatus(releaseId,
dataFileName,
IStatus.STAGE_1,
(double)totalRowCount / dataRows * 100);
}
}
if (errors.Count > 0)
{
return(errors);
}
await _importStatusService.UpdateStatus(releaseId,
dataFileName,
IStatus.STAGE_1,
100);
var rowsPerBatch = Convert.ToInt32(LoadAppSettings(executionContext).GetValue <string>("RowsPerBatch"));
return(new ProcessorStatistics
{
FilteredObservationCount = filteredRows,
RowsPerBatch = rowsPerBatch,
NumBatches = FileStorageUtils.GetNumBatches(totalRowCount, rowsPerBatch)
});
}