private static bool IsGeographicLevelIgnored(IReadOnlyList <string> line, List <string> headers) { var geographicLevel = ImporterService.GetGeographicLevel(line, headers); return(ImporterService.IgnoredGeographicLevels.Contains(geographicLevel)); }
ValidateAndCountObservations( DataColumnCollection cols, DataRowCollection rows, ExecutionContext executionContext, Guid releaseId, string dataFileName) { var idx = 0; var filteredRows = 0; var totalRowCount = 0; var errors = new List <ValidationError>(); var dataRows = rows.Count; foreach (DataRow row in rows) { idx++; if (errors.Count == 100) { errors.Add(new ValidationError(FirstOneHundredErrors.GetEnumLabel())); break; } try { var rowValues = CsvUtil.GetRowValues(row); var colValues = CsvUtil.GetColumnValues(cols); ImporterService.GetGeographicLevel(rowValues, colValues); ImporterService.GetTimeIdentifier(rowValues, colValues); ImporterService.GetYear(rowValues, colValues); if (!IsGeographicLevelIgnored(rowValues, colValues)) { filteredRows++; } } catch (Exception e) { errors.Add(new ValidationError($"error at row {idx}: {e.Message}")); } totalRowCount++; if (totalRowCount % STAGE_1_ROW_CHECK == 0) { await _importStatusService.UpdateStatus(releaseId, dataFileName, IStatus.STAGE_1, (double)totalRowCount / dataRows * 100); } } if (errors.Count > 0) { return(errors); } await _importStatusService.UpdateStatus(releaseId, dataFileName, IStatus.STAGE_1, 100); var rowsPerBatch = Convert.ToInt32(LoadAppSettings(executionContext).GetValue <string>("RowsPerBatch")); return(new ProcessorStatistics { FilteredObservationCount = filteredRows, RowsPerBatch = rowsPerBatch, NumBatches = FileStorageUtils.GetNumBatches(totalRowCount, rowsPerBatch) }); }