private async Task <Int32> PopulateWorkerQueueAsync(ImportManagerQueueRecord managerQueueRecord, IParser parser, IAdminObject migrationObject, List <ImportJobError> violations)
{
var loadFileColumns = parser.ParseColumns();
var workerQueue = new Dictionary <long, String>();
Int32 lineNumber = LongToInt(parser.LineNumber);
var timeOfInsert = DateTime.UtcNow;
try
{
while (parser.Read())
{
try
{
var adminObjectDictionary = new Dictionary <String, String>();
foreach (var column in loadFileColumns)
{
var value = parser[column] == null ? String.Empty : parser[column].ToString();
adminObjectDictionary.Add(column, value);
}
await migrationObject.MapObjectColumnsAsync(adminObjectDictionary);
var serializedJson = await _serializationHelper.SerializeAdminObjectAsync(migrationObject);
workerQueue.Add(lineNumber, serializedJson);
}
catch (Exception ex)
{
violations.Add(new ImportJobError()
{
Message = String.Format(Constant.ErrorMessages.ImportQueueManagerLineNumberError, lineNumber, ex.Message), LineNumber = lineNumber >= Int32.MaxValue ? null : (Int32?)lineNumber, Type = Constant.ImportUtilityJob.ErrorType.DataLevel, Details = ex.ToString()
});
}
if ((workerQueue.Count % Constant.BatchSizes.ImportManagerIntoWorkerQueue) == 0)
{
await WriteQueueRecordsAsync(managerQueueRecord, workerQueue, violations, timeOfInsert);
workerQueue.Clear();
}
//The parser returns the current line from the Read above and advances the cursor to the next line. We want our logs should report the current line number of data values so linenumber should be updated last.
//parser.LineNumber is -1 when the parser get to the end of the file and we don't want to report that value
if (parser.LineNumber != -1)
{
lineNumber = LongToInt(parser.LineNumber);
}
}
if (workerQueue.Any())
{
await WriteQueueRecordsAsync(managerQueueRecord, workerQueue, violations, timeOfInsert);
}
return((lineNumber - 1) < 0 ? 0 : lineNumber - 1); //minus 1 because the header row should not be counted
}
catch (Exception ex)
{
// This is thrown instead of being recorded in the violation list because parser errors indicate load file issues, the user should updated the file before attempting the job again
throw new AdminMigrationUtilityException(String.Format(Constant.ErrorMessages.UnableToParseLineNumberError, lineNumber), ex);
}
}
