Пример #1
0
            public bool MoveNext()
            {
                Current = default;
                // take any white-space
                while (_value.PeekByte() == (byte)' ')
                {
                    _value.Consume(1);
                }

                byte terminator = (byte)' ';
                var  first      = _value.PeekByte();

                if (first < 0)
                {
                    return(false);           // EOF
                }
                switch (first)
                {
                case (byte)'"':
                case (byte)'\'':
                    // start of string
                    terminator = (byte)first;
                    _value.Consume(1);
                    break;
                }

                int end = BufferReader.FindNext(_value, terminator);

                if (end < 0)
                {
                    Current = _value.ConsumeToEnd();
                }
                else
                {
                    Current = _value.ConsumeAsBuffer(end);
                    _value.Consume(1); // drop the terminator itself;
                }
                return(true);
            }
Пример #2
0
        internal static int FindNext(BufferReader reader, byte value) // very deliberately not ref; want snapshot
        {
            int totalSkipped = 0;

            do
            {
                if (reader.RemainingThisSpan == 0)
                {
                    continue;
                }

                var span  = reader.SlicedSpan;
                int found = span.VectorSafeIndexOf(value);
                if (found >= 0)
                {
                    return(totalSkipped + found);
                }

                totalSkipped += span.Length;
            } while (reader.FetchNextSegment());
            return(-1);
        }
Пример #3
0
        internal static int FindNextCrLf(BufferReader reader) // very deliberately not ref; want snapshot
        {
            // is it in the current span? (we need to handle the offsets differently if so)

            int  totalSkipped        = 0;
            bool haveTrailingCR      = false;
            ReadOnlySpan <byte> CRLF = stackalloc byte[2] {
                (byte)'\r', (byte)'\n'
            };

            do
            {
                if (reader.RemainingThisSpan == 0)
                {
                    continue;
                }

                var span = reader.SlicedSpan;
                if (haveTrailingCR)
                {
                    if (span[0] == '\n')
                    {
                        return(totalSkipped - 1);
                    }
                    haveTrailingCR = false;
                }

                int found = span.IndexOf(CRLF);
                if (found >= 0)
                {
                    return(totalSkipped + found);
                }

                haveTrailingCR = span[span.Length - 1] == '\r';
                totalSkipped  += span.Length;
            }while (reader.FetchNextSegment());
            return(-1);
        }
Пример #4
0
 public Tokenizer(ReadOnlySequence <byte> value)
 {
     _value  = new BufferReader(value);
     Current = default;
 }