public void TryReadTo_Sequence(bool advancePastDelimiter, bool useEscapeOverload)
{
ReadOnlySequence <byte> bytes = SequenceFactory.Create(new byte[][] {
new byte[] { 0 },
new byte[] { 1, 2 },
new byte[] { },
new byte[] { 3, 4, 5, 6 }
});
SequenceReader <byte> reader = new SequenceReader <byte>(bytes);
// Read to 0-5
for (byte i = 0; i < bytes.Length - 1; i++)
{
SequenceReader <byte> copy = reader;
// Can read to the first integer (0-5)
Assert.True(
useEscapeOverload
? copy.TryReadTo(out ReadOnlySequence <byte> sequence, i, 255, advancePastDelimiter)
: copy.TryReadTo(out sequence, i, advancePastDelimiter));
// Should never have a null Position object
Assert.NotNull(copy.Position.GetObject());
ReadOnlySequence <byte> .Enumerator enumerator = sequence.GetEnumerator();
while (enumerator.MoveNext())
{
;
}
// Should be able to read to final 6
Assert.True(
useEscapeOverload
? copy.TryReadTo(out sequence, 6, 255, advancePastDelimiter)
: copy.TryReadTo(out sequence, 6, advancePastDelimiter));
Assert.NotNull(copy.Position.GetObject());
enumerator = sequence.GetEnumerator();
while (enumerator.MoveNext())
{
;
}
// If we didn't advance, we should still be able to read to 6
Assert.Equal(!advancePastDelimiter,
useEscapeOverload
? copy.TryReadTo(out sequence, 6, 255, advancePastDelimiter)
: copy.TryReadTo(out sequence, 6, advancePastDelimiter));
}
}
public ReadOnlySequenceWrapper(ReadOnlySequence <byte> data)
{
_data = data;
_currentOffset = 0;
_totalOffset = 0;
_enumerator = data.GetEnumerator();
_enumerator.MoveNext();
}
private bool RefillFromReadOnlySequence(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state, bool mustSucceed)
{
CheckCurrentBufferIsEmpty(ref state);
if (state.totalBytesRetired + state.bufferSize == state.currentLimit)
{
// Oops, we hit a limit.
if (mustSucceed)
{
throw InvalidProtocolBufferException.TruncatedMessage();
}
else
{
return(false);
}
}
state.totalBytesRetired += state.bufferSize;
state.bufferPos = 0;
state.bufferSize = 0;
while (readOnlySequenceEnumerator.MoveNext())
{
buffer = readOnlySequenceEnumerator.Current.Span;
state.bufferSize = buffer.Length;
if (buffer.Length != 0)
{
break;
}
}
if (state.bufferSize == 0)
{
if (mustSucceed)
{
throw InvalidProtocolBufferException.TruncatedMessage();
}
else
{
return(false);
}
}
else
{
RecomputeBufferSizeAfterLimit(ref state);
int totalBytesRead =
state.totalBytesRetired + state.bufferSize + state.bufferSizeAfterLimit;
if (totalBytesRead < 0 || totalBytesRead > state.sizeLimit)
{
throw InvalidProtocolBufferException.SizeLimitExceeded();
}
return(true);
}
}
public long Seek(long offset, IntPtr whence, IntPtr user_data)
{
var newOffset = GetNewOffset(offset, whence);
if (newOffset < 0)
{
return(-1);
}
var steps = newOffset - _totalOffset;
if (steps > 0 && _currentOffset + steps < _enumerator.Current.Span.Length)
{
_currentOffset += steps;
_totalOffset = newOffset;
}
else
{
_totalOffset = -1;
_currentOffset = 0;
var remaining = newOffset;
try
{
_enumerator = _data.GetEnumerator();
while (_enumerator.MoveNext())
{
var current = _enumerator.Current.Span;
if (remaining < current.Length)
{
_currentOffset = remaining;
_totalOffset = newOffset;
break;
}
remaining -= current.Length;
}
}
catch
{
return(_totalOffset);
}
if (remaining == 0)
{
_totalOffset = newOffset;
}
}
return(_totalOffset);
}
private int ReadNextBuffer(ref State state, bool throwIfEOF)
{
do
{
if (!_source.MoveNext())
{
if (throwIfEOF)
{
ThrowEoF(this, ref state);
}
return(0);
}
state.Init(_source.Current);
} while (state.Span.IsEmpty);
return(state.Span.Length);
}
private bool FetchNextSegment()
{
do
{
if (!_iterator.MoveNext())
{
OffsetThisSpan = RemainingThisSpan = 0;
return(false);
}
_current = _iterator.Current.Span;
OffsetThisSpan = 0;
RemainingThisSpan = _current.Length;
} while (IsEmpty); // skip empty segments, they don't help us!
return(true);
}
public int Read(Span <char> into)
{
AssertNotDisposedInternal(this);
if (IsComplete)
{
return(0);
}
tryAgain:
if (CurrentSegment.IsEmpty)
{
if (!Enumerator.MoveNext())
{
IsComplete = true;
return(0);
}
CurrentSegment = Enumerator.Current;
goto tryAgain;
}
var intoLen = into.Length;
var curSegLen = CurrentSegment.Length;
if (intoLen > curSegLen)
{
// current segment can't fill into
CurrentSegment.Span.CopyTo(into);
CurrentSegment = ReadOnlyMemory <char> .Empty;
var remainingInto = into.Slice(curSegLen);
var nextChunkRs = Read(remainingInto);
return(curSegLen + nextChunkRs);
}
else
{
// current segment will fill into
CurrentSegment.Span.Slice(0, intoLen).CopyTo(into);
CurrentSegment = CurrentSegment.Slice(intoLen);
return(intoLen);
}
}
private static bool EatUTF8BOM(CsvTokenizer tokenizer, CsvReaderVisitorBase visitor, ref ReadOnlySequence <byte> .Enumerator enumerator)
{
ReadOnlyMemory <byte> segment;
while (true)
{
if (!enumerator.MoveNext())
{
tokenizer.ProcessEndOfStream(visitor);
return(true);
}
segment = enumerator.Current;
if (!segment.IsEmpty)
{
break;
}
}
var span = segment.Span;
ReadOnlySpan <byte> head = UTF8BOM;
// this greed should **probably** pay off most of the time.
if (span.Length >= 3)
{
if (span.StartsWith(head))
{
span = span.Slice(3);
}
tokenizer.ProcessNextChunk(span, visitor);
return(false);
}
int alreadyEaten = 0;
while (true)
{
if (span[0] == head[alreadyEaten])
{
span = span.Slice(1);
if (++alreadyEaten == 3)
{
tokenizer.ProcessNextChunk(span, visitor);
return(false);
}
}
else
{
tokenizer.ProcessNextChunk(head.Slice(0, alreadyEaten), visitor);
tokenizer.ProcessNextChunk(span, visitor);
return(false);
}
if (span.IsEmpty)
{
while (true)
{
if (!enumerator.MoveNext())
{
tokenizer.ProcessEndOfStream(visitor);
return(true);
}
segment = enumerator.Current;
if (!segment.IsEmpty)
{
break;
}
}
span = segment.Span;
}
}
}
public long Read(IntPtr data, UIntPtr count, IntPtr user_data)
{
var total = (long)count;
if (total == 0)
{
return(0);
}
if (data == IntPtr.Zero)
{
return(0);
}
var destination = (byte *)data.ToPointer();
long bytesRead = 0;
while (total > 0)
{
long length = 0;
ReadOnlySpan <byte> current;
try
{
current = _enumerator.Current.Span;
length = Math.Min(total, current.Length - _currentOffset);
fixed(byte *source = current)
{
NativeMemory.Copy(source + _currentOffset, destination, length);
}
}
catch
{
return(bytesRead);
}
_currentOffset += length;
_totalOffset += length;
bytesRead += length;
destination += length;
total -= length;
if (_currentOffset == current.Length)
{
try
{
if (!_enumerator.MoveNext())
{
break;
}
}
catch
{
return(-1);
}
_currentOffset = 0;
}
}
return(bytesRead);
}
public bool MoveNext() => _enumerator.MoveNext();