/// <summary> /// Simply logs progress every 25k records to the configured ILogger for this type /// </summary> public IEnumerable <IReadOnlyDictionary <string, string> > GetRecords(int limit = int.MaxValue) { var timer = Stopwatch.StartNew(); foreach (var record in _innerRecordReader.GetRecords(limit)) { if (_innerRecordReader.CountRead % _modCountToLogProgress == 0) { _logger.LogInformation($"Read [{_innerRecordReader.CountRead,0:N0}] records from source, [{_innerRecordReader.CountFail,0:N0}] failed reads, in [{timer.Elapsed:mm\\:ss}]"); } yield return(record); } timer.Stop(); _logger.LogInformation($"Read [{_innerRecordReader.CountRead - 1,0:N0}] total records from source, [{_innerRecordReader.CountFail,0:N0}] total failed reads, in [{timer.Elapsed:mm\\:ss}]"); }
public void Wrangle(int limit = int.MaxValue) { var stopWatch = Stopwatch.StartNew(); _logger.LogInformation($"Starting Wrangling for recordType [{_recordConfiguration.RecordTypeName}]"); // Simple lazy pipeline // Get source records, build target records, format the built targets, filter the final result, write the records... var countProcessed = _recordReader.GetRecords(limit) .Then(source => _recordBuilder.Build(source, _recordConfiguration)) .Then(built => _fieldFormatter.Format(built, _recordConfiguration)) .Then(format => _fieldFilterService.Filter(format, _recordConfiguration)) .Then(filter => _recordWriter.Write(filter, _recordConfiguration)) .Count(); stopWatch.Stop(); _logger.LogInformation($"Finished Wrangling, [{countProcessed}] target records successfully written, [{_recordReader.CountRead}] source reacords read, in [{stopWatch.Elapsed:mm\\:ss}]"); }