/// <summary>
/// Simply logs progress every 25k records to the configured ILogger for this type
/// </summary>
public IEnumerable <IReadOnlyDictionary <string, string> > GetRecords(int limit = int.MaxValue)
{
var timer = Stopwatch.StartNew();
foreach (var record in _innerRecordReader.GetRecords(limit))
{
if (_innerRecordReader.CountRead % _modCountToLogProgress == 0)
{
_logger.LogInformation($"Read [{_innerRecordReader.CountRead,0:N0}] records from source, [{_innerRecordReader.CountFail,0:N0}] failed reads, in [{timer.Elapsed:mm\\:ss}]");
}
yield return(record);
}
timer.Stop();
_logger.LogInformation($"Read [{_innerRecordReader.CountRead - 1,0:N0}] total records from source, [{_innerRecordReader.CountFail,0:N0}] total failed reads, in [{timer.Elapsed:mm\\:ss}]");
}
public void Wrangle(int limit = int.MaxValue)
{
var stopWatch = Stopwatch.StartNew();
_logger.LogInformation($"Starting Wrangling for recordType [{_recordConfiguration.RecordTypeName}]");
// Simple lazy pipeline
// Get source records, build target records, format the built targets, filter the final result, write the records...
var countProcessed = _recordReader.GetRecords(limit)
.Then(source => _recordBuilder.Build(source, _recordConfiguration))
.Then(built => _fieldFormatter.Format(built, _recordConfiguration))
.Then(format => _fieldFilterService.Filter(format, _recordConfiguration))
.Then(filter => _recordWriter.Write(filter, _recordConfiguration))
.Count();
stopWatch.Stop();
_logger.LogInformation($"Finished Wrangling, [{countProcessed}] target records successfully written, [{_recordReader.CountRead}] source reacords read, in [{stopWatch.Elapsed:mm\\:ss}]");
}
