/// <summary>
/// Writes a record.
/// </summary>
/// <param name="record">the record to write</param>
public void Write(byte[] record)
{
if (Filter == null || Filter.Select(record))
{
_writer.Write(Formatter == null ? record : Formatter.Format(record));
}
}
/// <summary>
/// Writes a record using the output formatter if it has been defined.
/// </summary>
/// <param name="record">the record to write</param>
private void WriteRecord(T record)
{
if (_logger.IsTraceEnabled)
{
_logger.Trace("Writing record: {0}", ObjectUtils.Dump(record));
}
_writer.Write(_outputFormatter == null ? record : _outputFormatter.Format(record));
}
public void Write(IRecordWriter writer, string value)
{
if (_maximumLength != null)
value = _maximumLength.Write(value);
if (_minimumLength != null)
value = _minimumLength.Write(value);
writer.Write(value, 0, _index);
}
public static void Copy(IRecordReader<string, string> reader, IRecordWriter<string, string> writer)
{
var keys = new List<string>(reader.GetKeys());
for (int i = 0; i < keys.Count; i++)
{
var key = keys[i];
if (i % 100 == 0 && i > 0)
Console.WriteLine("Copy key " + i + "/" + keys.Count);
var values = reader.GetValues(key);
foreach (var value in values)
writer.Write(key, value);
}
}
public void Write(IRecordWriter writer, string value)
{
if (_maximumLength != null)
{
value = _maximumLength.Write(value);
}
if (_minimumLength != null)
{
value = _minimumLength.Write(value);
}
writer.Write(value, 0, _index);
}
public void Write(IRecordWriter writer, string value)
{
if (_maximumLength != null)
{
value = _maximumLength.Write(value);
}
if (_minimumLength != null)
{
value = _minimumLength.Write(value);
}
writer.Write(value, _position, 0);
}
/// <summary>
/// Merges the temporary files to the final output file
/// </summary>
/// <param name="buffers">the buffers to read the temporary files</param>
/// <param name="writer">the writer for the output file</param>
private static void Merge(IEnumerable <RecordReaderBuffer <T> > buffers, IRecordWriter <T> writer)
{
// Buffers are stored in a priority queue to have the buffers with the
// lowest record (with respect to Comparer) as the first buffer
var queue = new PriorityQueue <RecordReaderBuffer <T> >(buffers);
while (queue.Count > 0)
{
var buffer = queue.Poll();
var record = buffer.Read();
writer.Write(record);
// If the buffer has still records, we put it back in the queue
// so that is is correctly sorted
if (buffer.HasNext())
{
queue.Add(buffer);
}
}
}
public void Wrangle(int limit = int.MaxValue)
{
var stopWatch = Stopwatch.StartNew();
_logger.LogInformation($"Starting Wrangling for recordType [{_recordConfiguration.RecordTypeName}]");
// Simple lazy pipeline
// Get source records, build target records, format the built targets, filter the final result, write the records...
var countProcessed = _recordReader.GetRecords(limit)
.Then(source => _recordBuilder.Build(source, _recordConfiguration))
.Then(built => _fieldFormatter.Format(built, _recordConfiguration))
.Then(format => _fieldFilterService.Filter(format, _recordConfiguration))
.Then(filter => _recordWriter.Write(filter, _recordConfiguration))
.Count();
stopWatch.Stop();
_logger.LogInformation($"Finished Wrangling, [{countProcessed}] target records successfully written, [{_recordReader.CountRead}] source reacords read, in [{stopWatch.Elapsed:mm\\:ss}]");
}
public void Write(IRecordWriter writer, string value)
{
value = String.Format("{0}={1}", _name, value); // is this a behavior? It could tell us how to write the value and how to match & parse on read
writer.Write(value, _position);
}
public void Write(IRecordWriter writer, string value)
{
if (_maximumLength != null)
value = _maximumLength.Write(value);
if (_minimumLength != null)
value = _minimumLength.Write(value);
writer.Write(value, _position, 0);
}
public void Write(IRecordWriter writer, string value)
{
value = String.Format("{0}={1}", _name, value); // is this a behavior? It could tell us how to write the value and how to match & parse on read
writer.Write(value, _position);
}