public static IEnumerable <LineConstructor> ReadLines(Stream stream, Encoding encoding, byte[] workingBuffer, byte[] separator, IMetricsHost metrics = null, CancellationToken cancellationToken = default) { var queue = new BlockingCollection <LineConstructor>(new ConcurrentQueue <LineConstructor>()); void ReadLines(Encoding e) { try { unsafe { LineReader.ReadLines(stream, e, workingBuffer, (lineNumber, start, length, x, m) => { // TODO convert to row buffer/allocator var buffer = new byte[length]; new ReadOnlySpan <byte>(start, length).CopyTo(buffer); var ctor = new LineConstructor { lineNumber = lineNumber, length = length, start = buffer }; queue.Add(ctor, cancellationToken); }, metrics, cancellationToken); } } catch (Exception ex) { throw ex; } finally { queue.CompleteAdding(); } } Task.Run(() => ReadLines(encoding), cancellationToken); return(queue.GetConsumingEnumerable()); }
public static IEnumerable <LineConstructor> StreamLines(Stream stream, Encoding encoding, byte[] workingBuffer, byte[] separator, int maxWorkingMemoryBytes = 0, IMetricsHost metrics = null, CancellationToken cancellationToken = default) { var queue = new BlockingCollection <LineConstructor>(new ConcurrentQueue <LineConstructor>()); void ReadLines(Encoding e) { var pendingLength = 0; byte[] buffer = null; // TODO convert to allocator try { unsafe { LineReader.ReadLines(stream, e, workingBuffer, (lineNumber, partial, start, length, x, m) => { if (buffer == null) { buffer = new byte[Math.Max(length, Constants.ReadAheadSize * 2)]; } var target = new Span <byte>(buffer, pendingLength, length); var segment = new ReadOnlySpan <byte>(start, length); segment.CopyTo(target); if (partial) { pendingLength = length; } else { var ctor = new LineConstructor { lineNumber = lineNumber, length = length + pendingLength, buffer = buffer }; if (maxWorkingMemoryBytes > 0) { var usedBytes = queue.Count * (buffer.Length + sizeof(long) + sizeof(int)); while (usedBytes > maxWorkingMemoryBytes) { Task.Delay(10, cancellationToken).Wait(cancellationToken); } } queue.Add(ctor, cancellationToken); pendingLength = 0; buffer = null; } }, metrics, cancellationToken); } } finally { queue.CompleteAdding(); } } Task.Run(() => ReadLines(encoding), cancellationToken); return(queue.GetConsumingEnumerable()); }