public bool MoveNext()
{
Current = default;
// take any white-space
while (_value.PeekByte() == (byte)' ')
{
_value.Consume(1);
}
byte terminator = (byte)' ';
var first = _value.PeekByte();
if (first < 0)
{
return(false); // EOF
}
switch (first)
{
case (byte)'"':
case (byte)'\'':
// start of string
terminator = (byte)first;
_value.Consume(1);
break;
}
int end = BufferReader.FindNext(_value, terminator);
if (end < 0)
{
Current = _value.ConsumeToEnd();
}
else
{
Current = _value.ConsumeAsBuffer(end);
_value.Consume(1); // drop the terminator itself;
}
return(true);
}
internal static int FindNext(BufferReader reader, byte value) // very deliberately not ref; want snapshot
{
int totalSkipped = 0;
do
{
if (reader.RemainingThisSpan == 0)
{
continue;
}
var span = reader.SlicedSpan;
int found = span.VectorSafeIndexOf(value);
if (found >= 0)
{
return(totalSkipped + found);
}
totalSkipped += span.Length;
} while (reader.FetchNextSegment());
return(-1);
}
internal static int FindNextCrLf(BufferReader reader) // very deliberately not ref; want snapshot
{
// is it in the current span? (we need to handle the offsets differently if so)
int totalSkipped = 0;
bool haveTrailingCR = false;
ReadOnlySpan <byte> CRLF = stackalloc byte[2] {
(byte)'\r', (byte)'\n'
};
do
{
if (reader.RemainingThisSpan == 0)
{
continue;
}
var span = reader.SlicedSpan;
if (haveTrailingCR)
{
if (span[0] == '\n')
{
return(totalSkipped - 1);
}
haveTrailingCR = false;
}
int found = span.IndexOf(CRLF);
if (found >= 0)
{
return(totalSkipped + found);
}
haveTrailingCR = span[span.Length - 1] == '\r';
totalSkipped += span.Length;
}while (reader.FetchNextSegment());
return(-1);
}
public Tokenizer(ReadOnlySequence <byte> value)
{
_value = new BufferReader(value);
Current = default;
}