public static IEnumerable <LineConstructor> ReadLines(Stream stream, Encoding encoding, byte[] workingBuffer,
byte[] separator, IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
var queue = new BlockingCollection <LineConstructor>(new ConcurrentQueue <LineConstructor>());
void ReadLines(Encoding e)
{
try
{
unsafe
{
LineReader.ReadLines(stream, e, workingBuffer, (lineNumber, start, length, x, m) =>
{
// TODO convert to row buffer/allocator
var buffer = new byte[length];
new ReadOnlySpan <byte>(start, length).CopyTo(buffer);
var ctor = new LineConstructor
{
lineNumber = lineNumber,
length = length,
start = buffer
};
queue.Add(ctor, cancellationToken);
}, metrics, cancellationToken);
}
}
catch (Exception ex)
{
throw ex;
}
finally
{
queue.CompleteAdding();
}
}
Task.Run(() => ReadLines(encoding), cancellationToken);
return(queue.GetConsumingEnumerable());
}
public static IEnumerable <LineConstructor> StreamLines(Stream stream, Encoding encoding, byte[] workingBuffer,
byte[] separator, int maxWorkingMemoryBytes = 0, IMetricsHost metrics = null,
CancellationToken cancellationToken = default)
{
var queue = new BlockingCollection <LineConstructor>(new ConcurrentQueue <LineConstructor>());
void ReadLines(Encoding e)
{
var pendingLength = 0;
byte[] buffer = null; // TODO convert to allocator
try
{
unsafe
{
LineReader.ReadLines(stream, e, workingBuffer, (lineNumber, partial, start, length, x, m) =>
{
if (buffer == null)
{
buffer = new byte[Math.Max(length, Constants.ReadAheadSize * 2)];
}
var target = new Span <byte>(buffer, pendingLength, length);
var segment = new ReadOnlySpan <byte>(start, length);
segment.CopyTo(target);
if (partial)
{
pendingLength = length;
}
else
{
var ctor = new LineConstructor
{
lineNumber = lineNumber, length = length + pendingLength, buffer = buffer
};
if (maxWorkingMemoryBytes > 0)
{
var usedBytes = queue.Count * (buffer.Length + sizeof(long) + sizeof(int));
while (usedBytes > maxWorkingMemoryBytes)
{
Task.Delay(10, cancellationToken).Wait(cancellationToken);
}
}
queue.Add(ctor, cancellationToken);
pendingLength = 0;
buffer = null;
}
}, metrics, cancellationToken);
}
}
finally
{
queue.CompleteAdding();
}
}
Task.Run(() => ReadLines(encoding), cancellationToken);
return(queue.GetConsumingEnumerable());
}