private static void CopyRows(DataTable target, IEnumerable <DataRow> rows, List <string> headerList)
{
foreach (var row in from line in rows let s = CsvUtil.GetRowValues(line) where !IsGeographicLevelIgnored(s, headerList) select line)
{
target.Rows.Add(row.ItemArray);
}
}
private static void CopyRows(DataTable target,
IEnumerable <DataRow> rows,
List <string> colValues,
bool hasSoleGeographicLevel)
{
rows.ForEach(row =>
{
var rowValues = CsvUtil.GetRowValues(row);
if (CsvUtil.IsRowAllowed(hasSoleGeographicLevel, rowValues, colValues))
{
target.Rows.Add(row.ItemArray);
}
});
}
public static MetaRow GetMetaRow(List <string> cols, DataRow row)
{
return(CsvUtil.BuildType(CsvUtil.GetRowValues(row),
cols, Enum.GetNames(typeof(MetaColumns)), values => new MetaRow
{
ColumnName = values[0],
ColumnType = (ColumnType)Enum.Parse(typeof(ColumnType), values[1]),
Label = values[2],
FilterGroupingColumn = values[3],
FilterHint = values[4],
IndicatorGrouping = values[5],
IndicatorUnit = EnumUtil.GetFromString <Unit>(values[6] ?? ""),
DecimalPlaces = values[7] == null ? (int?)null : int.Parse(values[7])
}));
}
ValidateAndCountObservations(
DataColumnCollection cols,
DataRowCollection rows,
ExecutionContext executionContext,
Guid releaseId,
string dataFileName)
{
var idx = 0;
var filteredRows = 0;
var totalRowCount = 0;
var errors = new List <ValidationError>();
var dataRows = rows.Count;
foreach (DataRow row in rows)
{
idx++;
if (errors.Count == 100)
{
errors.Add(new ValidationError(FirstOneHundredErrors.GetEnumLabel()));
break;
}
try
{
var rowValues = CsvUtil.GetRowValues(row);
var colValues = CsvUtil.GetColumnValues(cols);
ImporterService.GetGeographicLevel(rowValues, colValues);
ImporterService.GetTimeIdentifier(rowValues, colValues);
ImporterService.GetYear(rowValues, colValues);
if (!IsGeographicLevelIgnored(rowValues, colValues))
{
filteredRows++;
}
}
catch (Exception e)
{
errors.Add(new ValidationError($"error at row {idx}: {e.Message}"));
}
totalRowCount++;
if (totalRowCount % STAGE_1_ROW_CHECK == 0)
{
await _importStatusService.UpdateStatus(releaseId,
dataFileName,
IStatus.STAGE_1,
(double)totalRowCount / dataRows * 100);
}
}
if (errors.Count > 0)
{
return(errors);
}
await _importStatusService.UpdateStatus(releaseId,
dataFileName,
IStatus.STAGE_1,
100);
var rowsPerBatch = Convert.ToInt32(LoadAppSettings(executionContext).GetValue <string>("RowsPerBatch"));
return(new ProcessorStatistics
{
FilteredObservationCount = filteredRows,
RowsPerBatch = rowsPerBatch,
NumBatches = FileStorageUtils.GetNumBatches(totalRowCount, rowsPerBatch)
});
}