public ParseResult ParseMessage(ReadOnlySequence <byte> buffer, out SequencePosition consumed, out SequencePosition examined, out byte[]?message)
{
consumed = buffer.Start;
examined = buffer.End;
message = null;
var start = consumed;
while (buffer.Length > 0)
{
if (!(buffer.PositionOf(ByteLF) is SequencePosition lineEnd))
{
// For the case of data: Foo\r\n\r\<Anything except \n>
if (_internalParserState == InternalParseState.ReadEndOfMessage)
{
if (ConvertBufferToSpan(buffer.Slice(start, buffer.End)).Length > 1)
{
throw new FormatException("Expected a \\r\\n frame ending");
}
}
// Partial message. We need to read more.
return(ParseResult.Incomplete);
}
lineEnd = buffer.GetPosition(1, lineEnd);
var line = ConvertBufferToSpan(buffer.Slice(start, lineEnd));
buffer = buffer.Slice(line.Length);
if (line.Length <= 1)
{
throw new FormatException("There was an error in the frame format");
}
// Skip comments
if (line[0] == ByteColon)
{
start = lineEnd;
consumed = lineEnd;
continue;
}
if (IsMessageEnd(line))
{
_internalParserState = InternalParseState.ReadEndOfMessage;
}
// To ensure that the \n was preceded by a \r
// since messages can't contain \n.
// data: foo\n\bar should be encoded as
// data: foo\r\n
// data: bar\r\n
else if (line[line.Length - SseLineEnding.Length] != ByteCR)
{
throw new FormatException("Unexpected '\\n' in message. A '\\n' character can only be used as part of the newline sequence '\\r\\n'");
}
else
{
EnsureStartsWithDataPrefix(line);
}
var payload = Array.Empty <byte>();
switch (_internalParserState)
{
case InternalParseState.ReadMessagePayload:
EnsureStartsWithDataPrefix(line);
// Slice away the 'data: '
var payloadLength = line.Length - (DataPrefix.Length + SseLineEnding.Length);
var newData = line.Slice(DataPrefix.Length, payloadLength).ToArray();
_data.Add(newData);
start = lineEnd;
consumed = lineEnd;
break;
case InternalParseState.ReadEndOfMessage:
if (_data.Count == 1)
{
payload = _data[0];
}
else if (_data.Count > 1)
{
// Find the final size of the payload
var payloadSize = 0;
foreach (var dataLine in _data)
{
payloadSize += dataLine.Length;
}
payloadSize += _newLine.Length * _data.Count;
// Allocate space in the payload buffer for the data and the new lines.
// Subtract newLine length because we don't want a trailing newline.
payload = new byte[payloadSize - _newLine.Length];
var offset = 0;
foreach (var dataLine in _data)
{
dataLine.CopyTo(payload, offset);
offset += dataLine.Length;
if (offset < payload.Length)
{
_newLine.CopyTo(payload, offset);
offset += _newLine.Length;
}
}
}
message = payload;
consumed = lineEnd;
examined = consumed;
return(ParseResult.Completed);
}
if (buffer.Length > 0 && buffer.First.Span[0] == ByteCR)
{
_internalParserState = InternalParseState.ReadEndOfMessage;
}
}
return(ParseResult.Incomplete);
}
private async ValueTask <MessageId> Send(PulsarApi.MessageMetadata metadata, ReadOnlySequence <byte> data, CancellationToken cancellationToken)
{
var response = await _channel.Send(metadata, data, cancellationToken).ConfigureAwait(false);
return(response.MessageId.ToMessageId());
}
public void TryReadTo_SkipDelimiter_Runs()
{
ReadOnlySequence <byte> bytes = SequenceFactory.CreateUtf8("abc^^|def");
SequenceReader <byte> reader = new SequenceReader <byte>(bytes);
Assert.True(reader.TryReadTo(out ReadOnlySpan <byte> span, (byte)'|', (byte)'^', advancePastDelimiter: false));
Assert.Equal(Encoding.UTF8.GetBytes("abc^^"), span.ToArray());
Assert.True(reader.IsNext((byte)'|'));
Assert.Equal(5, reader.Consumed);
// Split after escape char
bytes = SequenceFactory.CreateUtf8("abc^^", "|def");
reader = new SequenceReader <byte>(bytes);
Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false));
Assert.Equal(Encoding.UTF8.GetBytes("abc^^"), span.ToArray());
Assert.True(reader.IsNext((byte)'|'));
Assert.Equal(5, reader.Consumed);
// Split before and after escape char
bytes = SequenceFactory.CreateUtf8("abc^", "^", "|def");
reader = new SequenceReader <byte>(bytes);
Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false));
Assert.Equal(Encoding.UTF8.GetBytes("abc^^"), span.ToArray());
Assert.True(reader.IsNext((byte)'|'));
Assert.Equal(5, reader.Consumed);
// Check advance past delimiter
reader = new SequenceReader <byte>(bytes);
Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true));
Assert.Equal(Encoding.UTF8.GetBytes("abc^^"), span.ToArray());
Assert.True(reader.IsNext((byte)'d'));
Assert.Equal(6, reader.Consumed);
// Leading run of 2
bytes = SequenceFactory.CreateUtf8("^^|abc");
reader = new SequenceReader <byte>(bytes);
Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false));
Assert.Equal(Encoding.UTF8.GetBytes("^^"), span.ToArray());
Assert.True(reader.IsNext((byte)'|'));
Assert.Equal(2, reader.Consumed);
// Leading run of 3
bytes = SequenceFactory.CreateUtf8("^^^|abc");
reader = new SequenceReader <byte>(bytes);
Assert.False(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false));
Assert.True(reader.IsNext((byte)'^'));
Assert.Equal(0, reader.Consumed);
// Trailing run of 3
bytes = SequenceFactory.CreateUtf8("abc^^^|");
reader = new SequenceReader <byte>(bytes);
Assert.False(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false));
Assert.True(reader.IsNext((byte)'a'));
Assert.Equal(0, reader.Consumed);
// Trailing run of 3, split
bytes = SequenceFactory.CreateUtf8("abc^^^", "|");
reader = new SequenceReader <byte>(bytes);
Assert.False(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false));
Assert.True(reader.IsNext((byte)'a'));
Assert.Equal(0, reader.Consumed);
}
private HttpHeader ParseHttpHeaderItems(ReadOnlySequence <byte> header)
{
var headerText = header.GetString(Encoding.UTF8);
var reader = new StringReader(headerText);
var firstLine = reader.ReadLine();
if (string.IsNullOrEmpty(firstLine))
{
return(null);
}
var items = new NameValueCollection();
var prevKey = string.Empty;
var line = string.Empty;
while (!string.IsNullOrEmpty(line = reader.ReadLine()))
{
if (line.StartsWith(_TAB) && !string.IsNullOrEmpty(prevKey))
{
var currentValue = items.Get(prevKey);
items[prevKey] = currentValue + line.Trim();
continue;
}
int pos = line.IndexOf(_COLON);
if (pos <= 0)
{
continue;
}
string key = line.Substring(0, pos);
if (!string.IsNullOrEmpty(key))
{
key = key.Trim();
}
if (string.IsNullOrEmpty(key))
{
continue;
}
var valueOffset = pos + 1;
if (line.Length <= valueOffset) //No value in this line
{
continue;
}
var value = line.Substring(valueOffset);
if (!string.IsNullOrEmpty(value) && value.StartsWith(' ') && value.Length > 1)
{
value = value.Substring(1);
}
var existingValue = items.Get(key);
if (string.IsNullOrEmpty(existingValue))
{
items.Add(key, value);
}
else
{
items[key] = existingValue + ", " + value;
}
prevKey = key;
}
var metaInfo = firstLine.Split(' ');
if (metaInfo.Length != 3)
{
// invalid first line
return(null);
}
return(new HttpHeader(metaInfo[0], metaInfo[1], metaInfo[2], items));
}
public IEnumerable <Dictionary <string, dynamic> > Parse()
{
using (FileStream fs = new FileStream(_fullFilePath, FileMode.Open, FileAccess.Read))
{
bool canRead = true;
bool isFirst = true;
while (canRead)
{
byte[] chunk = new byte[CHUNK_SIZE];
if (isFirst)
{
_offset = 0;
}
else
{
_offset = 1;
chunk[0] = 91;
while (inBuffer != 0)
{
inBuffer--;
chunk[_offset] = _buffer[inBuffer];
_offset++;
}
}
var count = fs.Read(chunk, _offset, CHUNK_SIZE - _offset - 1);
isFirst = false;
if (count < CHUNK_SIZE - _offset - 1)
{
canRead = false;
}
for (int i = count + _offset - 1; i > 0; i--)
{
if (chunk[i] != 125)
{
_buffer[inBuffer] = chunk[i];
inBuffer++;
}
else
{
chunk[i + 1] = 93;
break;
}
}
if (_offset > 0)
{
for (int i = 1; i < CHUNK_SIZE; i++)
{
if (chunk[i] != 123)
{
chunk[i] = 32;
}
else
{
break;
}
}
}
ReadOnlySequence <byte> bytesSpan = new ReadOnlySequence <byte>(chunk);
Utf8JsonReader jsonReader = new Utf8JsonReader(bytesSpan);
Dictionary <string, dynamic> dataRows = JsonSerializer.Deserialize <Dictionary <string, dynamic> >(ref jsonReader, _serializerOptions);
yield return(dataRows);
}
}
yield return(_dataBlock);
}
internal bool NeedsEncode(ReadOnlySequence <char> head) => Utils.FindNeedsEncode(head, 0, Config) != -1;
public override bool TryParseMessage(ref ReadOnlySequence <byte> buffer, out ServerlessMessage message) => ServerlessProtocol.TryParseMessage(ref buffer, out message);
public static void ReadZeroCharacters()
{
using var tr = new ReadOnlySequence <char>(CharData).AsTextReader();
Equal(0, tr.Read(new char[0], 0, 0));
}
protected abstract int GetBodyLengthFromHeader(ReadOnlySequence <byte> buffer);
internal override ValueTask WriteAsync(ReadOnlySequence <byte> buffers, CancellationToken cancellationToken = default)
{
return(WriteAsync(buffers, endStream: false, cancellationToken));
}
public bool Verify(ReadOnlySequence <byte> eventBuffer, ReadOnlySequence <byte> checksumBuffer)
{
throw new System.NotImplementedException();
}
public void TestFilterDifferentSingleMark()
{
ListenOptions channelOptions = new ListenOptions();
List <byte> data = new List <byte>();
List <byte> allData = new List <byte>();
ReadOnlySequence <byte> sequence;
FilterInfo filterInfo = CreateFilterInfo(9);
IFilter pipelineFilter = new FixHeaderSizePipelineFilter(channelOptions.ChannelOptions, filterInfo);
BaizeSession baizeSession = new BaizeSession(Guid.NewGuid().ToString());
data.AddRange(new byte[] { (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4', (byte)'5', (byte)'6', (byte)0x02, (byte)0x00, (byte)'9', (byte)('1') });
sequence = new ReadOnlySequence <byte>(data.ToArray());
if (pipelineFilter.FilterData(baizeSession, sequence))
{
string str = baizeSession.Data.GetString(Encoding.ASCII);
Assert.Equal(Encoding.ASCII.GetString(data.ToArray()), str);
}
data.Clear();
data.AddRange(new byte[] { (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4', (byte)'5', (byte)'6', (byte)0x04, (byte)0x00, (byte)'9', (byte)('1') });
allData.AddRange(data);
sequence = new ReadOnlySequence <byte>(data.ToArray());
Assert.False(pipelineFilter.FilterData(baizeSession, sequence));
data.Clear();
data.AddRange(new byte[] { (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4' });
allData.AddRange(data);
sequence = new ReadOnlySequence <byte>(data.ToArray());
if (pipelineFilter.FilterData(baizeSession, sequence))
{
string str = baizeSession.Data.GetString(Encoding.ASCII);
string contentStr = Encoding.ASCII.GetString(allData.GetRange(0, 13).ToArray());
Assert.Equal(contentStr, str);
}
data.Clear();
data.AddRange(new byte[] { (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4', (byte)'5', (byte)'6', (byte)0x04 });
allData.AddRange(data);
sequence = new ReadOnlySequence <byte>(data.ToArray());
Assert.False(pipelineFilter.FilterData(baizeSession, sequence));
data.Clear();
data.AddRange(new byte[] { (byte)0x00, (byte)'9', (byte)('1'), (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4' });
allData.AddRange(data);
sequence = new ReadOnlySequence <byte>(data.ToArray());
if (pipelineFilter.FilterData(baizeSession, sequence))
{
string str = baizeSession.Data.GetString(Encoding.ASCII);
string contentStr = Encoding.ASCII.GetString(allData.GetRange(0, 13).ToArray());
Assert.Equal(contentStr, str);
allData.Clear();
}
data.Clear();
data.AddRange(new byte[] { (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4', (byte)'5', (byte)'6' });
allData.AddRange(data);
sequence = new ReadOnlySequence <byte>(data.ToArray());
Assert.False(pipelineFilter.FilterData(baizeSession, sequence));
data.Clear();
data.AddRange(new byte[] { (byte)0x04, (byte)0x00, (byte)'9', (byte)('1'), (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4' });
allData.AddRange(data);
sequence = new ReadOnlySequence <byte>(data.ToArray());
if (pipelineFilter.FilterData(baizeSession, sequence))
{
string str = baizeSession.Data.GetString(Encoding.ASCII);
string contentStr = Encoding.ASCII.GetString(allData.GetRange(0, 13).ToArray());
Assert.Equal(contentStr, str);
allData.Clear();
}
//测试包头长度在中间段
data.Clear();
data.AddRange(new byte[] { (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4', (byte)'5', (byte)'6' });
allData.AddRange(data);
sequence = new ReadOnlySequence <byte>(data.ToArray());
Assert.False(pipelineFilter.FilterData(baizeSession, sequence));
data.Clear();
data.AddRange(new byte[] { (byte)0x04, (byte)0x00, (byte)'9', (byte)('1') });
allData.AddRange(data);
sequence = new ReadOnlySequence <byte>(data.ToArray());
Assert.False(pipelineFilter.FilterData(baizeSession, sequence));
data.Clear();
data.AddRange(new byte[] { (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4' });
allData.AddRange(data);
sequence = new ReadOnlySequence <byte>(data.ToArray());
if (pipelineFilter.FilterData(baizeSession, sequence))
{
string str = baizeSession.Data.GetString(Encoding.ASCII);
string contentStr = Encoding.ASCII.GetString(allData.GetRange(0, 13).ToArray());
Assert.Equal(contentStr, str);
allData.Clear();
}
}
public int GetBodyLength(ReadOnlySequence <byte> sequence)
{
int rtn = sequence.GetInt16(7);
return(rtn);
}
private Message DecodeMessage(FramingConnection connection, ref ReadOnlySequence <byte> buffer)
{
// TODO: plumb through from binding
int maxBufferSize = TransportDefaults.MaxBufferSize;
var decoder = connection.ServerSessionDecoder;
while (!connection.EOF && buffer.Length > 0)
{
int bytesRead = decoder.Decode(buffer);
if (bytesRead > 0)
{
if (!connection.EnvelopeBuffer.IsEmpty)
{
var remainingEnvelopeBuffer = connection.EnvelopeBuffer.Slice(connection.EnvelopeOffset, connection.EnvelopeSize - connection.EnvelopeOffset);
CopyBuffer(buffer, remainingEnvelopeBuffer, bytesRead);
connection.EnvelopeOffset += bytesRead;
}
buffer = buffer.Slice(bytesRead);
}
switch (decoder.CurrentState)
{
case ServerSessionDecoder.State.EnvelopeStart:
int envelopeSize = decoder.EnvelopeSize;
if (envelopeSize > maxBufferSize)
{
// TODO: Sending faults
//base.SendFault(FramingEncodingString.MaxMessageSizeExceededFault, timeout);
throw DiagnosticUtility.ExceptionUtility.ThrowHelperError(
MaxMessageSizeStream.CreateMaxReceivedMessageSizeExceededException(maxBufferSize));
}
connection.EnvelopeBuffer = connection.BufferManager.TakeBuffer(envelopeSize);
connection.EnvelopeSize = envelopeSize;
connection.EnvelopeOffset = 0;
break;
case ServerSessionDecoder.State.EnvelopeEnd:
if (!connection.EnvelopeBuffer.IsEmpty)
{
Message message = null;
try
{
message = connection.MessageEncoder.ReadMessage(
new ArraySegment <byte>(connection.EnvelopeBuffer.ToArray(), 0, connection.EnvelopeSize),
connection.BufferManager,
connection.ServerSessionDecoder.ContentType);
}
catch (XmlException xmlException)
{
throw DiagnosticUtility.ExceptionUtility.ThrowHelperError(
new ProtocolException(SR.MessageXmlProtocolError, xmlException));
}
connection.EnvelopeBuffer = null;
return(message);
}
break;
case ServerSessionDecoder.State.End:
connection.EOF = true;
break;
}
}
return(null);
}
public async Task SocketCanReadAndWrite()
{
var loop = new UvLoopHandle(_logger);
loop.Init(_uv);
var tcp = new UvTcpHandle(_logger);
tcp.Init(loop, (a, b) => { });
var endPoint = new IPEndPoint(IPAddress.Loopback, 0);
tcp.Bind(endPoint);
var port = tcp.GetSockIPEndPoint().Port;
tcp.Listen(10, (_, status, error, state) =>
{
var tcp2 = new UvTcpHandle(_logger);
tcp2.Init(loop, (a, b) => { });
tcp.Accept(tcp2);
var data = Marshal.AllocCoTaskMem(500);
tcp2.ReadStart(
(a, b, c) => tcp2.Libuv.buf_init(data, 500),
async(__, nread, state2) =>
{
if (nread <= 0)
{
tcp2.Dispose();
}
else
{
for (var x = 0; x < 2; x++)
{
var req = new UvWriteReq(_logger);
req.DangerousInit(loop);
var block = new ReadOnlySequence <byte>(new byte[] { 65, 66, 67, 68, 69 });
await req.WriteAsync(
tcp2,
block);
}
}
},
null);
tcp.Dispose();
}, null);
var t = Task.Run(async() =>
{
var socket = TestConnection.CreateConnectedLoopbackSocket(port);
await socket.SendAsync(new[] { new ArraySegment <byte>(new byte[] { 1, 2, 3, 4, 5 }) },
SocketFlags.None);
socket.Shutdown(SocketShutdown.Send);
var buffer = new ArraySegment <byte>(new byte[2048]);
while (true)
{
var count = await socket.ReceiveAsync(new[] { buffer }, SocketFlags.None);
if (count <= 0)
{
break;
}
}
socket.Dispose();
});
loop.Run();
loop.Dispose();
await t;
}
/// <summary> /// Initializes a new binary DTO. /// </summary> /// <param name="content">The content of the object.</param> public BinaryTransferObject(ReadOnlySequence <byte> content) => Content = content;
protected override unsafe void UploadIndices <T>(ReadOnlySequence <T> indices)
{
UploadBuffer(ref indexBuffer, GLEnum.ELEMENT_ARRAY_BUFFER, indices, ref indexBufferSize);
}
protected override TextPackageInfo DecodePackage(ref ReadOnlySequence <byte> buffer)
{
return(new TextPackageInfo {
Text = buffer.GetString(Utf8Encoding)
});
}
public IKzWriter Add(ReadOnlySequence <byte> data)
{
_alg.TransformBlock(data); return(this);
}
// For multi-segment parsing of a read only sequence
private void ParseValuesSlow(
ref ReadOnlySequence <byte> buffer,
ref KeyValueAccumulator accumulator,
bool isFinalBlock)
{
var sequenceReader = new SequenceReader <byte>(buffer);
ReadOnlySequence <byte> keyValuePair;
var consumed = sequenceReader.Position;
var consumedBytes = default(long);
var equalsDelimiter = GetEqualsForEncoding();
var andDelimiter = GetAndForEncoding();
while (!sequenceReader.End)
{
if (!sequenceReader.TryReadTo(out keyValuePair, andDelimiter))
{
if (!isFinalBlock)
{
// Don't buffer indefinately
if ((uint)(sequenceReader.Consumed - consumedBytes) > (uint)KeyLengthLimit + (uint)ValueLengthLimit)
{
ThrowKeyOrValueTooLargeException();
}
break;
}
// This must be the final key=value pair
keyValuePair = buffer.Slice(sequenceReader.Position);
sequenceReader.Advance(keyValuePair.Length);
}
if (keyValuePair.IsSingleSegment)
{
ParseFormValuesFast(keyValuePair.FirstSpan, ref accumulator, isFinalBlock: true, out var segmentConsumed);
Debug.Assert(segmentConsumed == keyValuePair.FirstSpan.Length);
consumedBytes = sequenceReader.Consumed;
consumed = sequenceReader.Position;
continue;
}
var keyValueReader = new SequenceReader <byte>(keyValuePair);
ReadOnlySequence <byte> value;
if (keyValueReader.TryReadTo(out var key, equalsDelimiter))
{
if (key.Length > KeyLengthLimit)
{
ThrowKeyTooLargeException();
}
value = keyValuePair.Slice(keyValueReader.Position);
if (value.Length > ValueLengthLimit)
{
ThrowValueTooLargeException();
}
}
else
{
// Too long for the whole segment to be a key.
if (keyValuePair.Length > KeyLengthLimit)
{
ThrowKeyTooLargeException();
}
// There is no more data, this segment must be "key" with no equals or value.
key = keyValuePair;
value = default;
}
var decodedKey = GetDecodedStringFromReadOnlySequence(key);
var decodedValue = GetDecodedStringFromReadOnlySequence(value);
AppendAndVerify(ref accumulator, decodedKey, decodedValue);
consumedBytes = sequenceReader.Consumed;
consumed = sequenceReader.Position;
}
buffer = buffer.Slice(consumed);
}
private void InsertData(byte[] data)
{
_buffer = new ReadOnlySequence <byte>(data);
}
internal abstract bool VerifySignature(Span <byte> data, ReadOnlySequence <byte> signature);
KzScript(ReadOnlySequence <byte> script) : this()
{
_script = script;
}
public ReadOnlySequenceContent(ReadOnlySequence <byte> bytes) => _bytes = bytes;
public bool TryParseMessage(ref ReadOnlySequence <byte> input, IInvocationBinder binder, out HubMessage message)
{
throw new NotSupportedException();
}
public static HttpPipelineRequestContent Create(ReadOnlySequence <byte> bytes) => new ReadOnlySequenceContent(bytes);
private static bool TryParseValue(ref Utf8JsonReader reader, out JsonDocument document, bool shouldThrow)
{
JsonReaderState state = reader.CurrentState;
CheckSupportedOptions(state.Options, nameof(reader));
// Value copy to overwrite the ref on an exception and undo the destructive reads.
Utf8JsonReader restore = reader;
ReadOnlySpan <byte> valueSpan = default;
ReadOnlySequence <byte> valueSequence = default;
try
{
switch (reader.TokenType)
{
// A new reader was created and has never been read,
// so we need to move to the first token.
// (or a reader has terminated and we're about to throw)
case JsonTokenType.None:
// Using a reader loop the caller has identified a property they wish to
// hydrate into a JsonDocument. Move to the value first.
case JsonTokenType.PropertyName:
{
if (!reader.Read())
{
if (shouldThrow)
{
ThrowHelper.ThrowJsonReaderException(
ref reader,
ExceptionResource.ExpectedJsonTokens);
}
reader = restore;
document = null;
return(false);
}
break;
}
}
switch (reader.TokenType)
{
// Any of the "value start" states are acceptable.
case JsonTokenType.StartObject:
case JsonTokenType.StartArray:
{
long startingOffset = reader.TokenStartIndex;
// Placeholder until reader.Skip() is written (#33295)
{
int depth = reader.CurrentDepth;
// CurrentDepth rises late and falls fast,
// a payload of "[ 1, 2, 3, 4 ]" will report post-Read()
// CurrentDepth values of { 0, 1, 1, 1, 1, 0 },
// Since we're logically at 0 ([), Read() once and keep
// reading until we've come back down to 0 (]).
do
{
if (!reader.Read())
{
if (shouldThrow)
{
ThrowHelper.ThrowJsonReaderException(
ref reader,
ExceptionResource.ExpectedJsonTokens);
}
reader = restore;
document = null;
return(false);
}
} while (reader.CurrentDepth > depth);
}
long totalLength = reader.BytesConsumed - startingOffset;
ReadOnlySequence <byte> sequence = reader.OriginalSequence;
if (sequence.IsEmpty)
{
valueSpan = reader.OriginalSpan.Slice(
checked ((int)startingOffset),
checked ((int)totalLength));
}
else
{
valueSequence = sequence.Slice(startingOffset, totalLength);
}
Debug.Assert(
reader.TokenType == JsonTokenType.EndObject ||
reader.TokenType == JsonTokenType.EndArray);
break;
}
// Single-token values
case JsonTokenType.Number:
case JsonTokenType.True:
case JsonTokenType.False:
case JsonTokenType.Null:
{
if (reader.HasValueSequence)
{
valueSequence = reader.ValueSequence;
}
else
{
valueSpan = reader.ValueSpan;
}
break;
}
// String's ValueSequence/ValueSpan omits the quotes, we need them back.
case JsonTokenType.String:
{
ReadOnlySequence <byte> sequence = reader.OriginalSequence;
if (sequence.IsEmpty)
{
// Since the quoted string fit in a ReadOnlySpan originally
// the contents length plus the two quotes can't overflow.
int payloadLength = reader.ValueSpan.Length + 2;
Debug.Assert(payloadLength > 1);
ReadOnlySpan <byte> readerSpan = reader.OriginalSpan;
Debug.Assert(
readerSpan[(int)reader.TokenStartIndex] == (byte)'"',
$"Calculated span starts with {readerSpan[(int)reader.TokenStartIndex]}");
Debug.Assert(
readerSpan[(int)reader.TokenStartIndex + payloadLength - 1] == (byte)'"',
$"Calculated span ends with {readerSpan[(int)reader.TokenStartIndex + payloadLength - 1]}");
valueSpan = readerSpan.Slice((int)reader.TokenStartIndex, payloadLength);
}
else
{
long payloadLength = 2;
if (reader.HasValueSequence)
{
payloadLength += reader.ValueSequence.Length;
}
else
{
payloadLength += reader.ValueSpan.Length;
}
valueSequence = sequence.Slice(reader.TokenStartIndex, payloadLength);
Debug.Assert(
valueSequence.First.Span[0] == (byte)'"',
$"Calculated sequence starts with {valueSequence.First.Span[0]}");
Debug.Assert(
valueSequence.ToArray()[payloadLength - 1] == (byte)'"',
$"Calculated sequence ends with {valueSequence.ToArray()[payloadLength - 1]}");
}
break;
}
default:
{
if (shouldThrow)
{
byte displayByte;
if (reader.HasValueSequence)
{
displayByte = reader.ValueSequence.First.Span[0];
}
else
{
displayByte = reader.ValueSpan[0];
}
ThrowHelper.ThrowJsonReaderException(
ref reader,
ExceptionResource.ExpectedStartOfValueNotFound,
displayByte);
}
reader = restore;
document = null;
return(false);
}
}
}
catch
{
reader = restore;
throw;
}
int length = valueSpan.IsEmpty ? checked ((int)valueSequence.Length) : valueSpan.Length;
byte[] rented = ArrayPool <byte> .Shared.Rent(length);
Span <byte> rentedSpan = rented.AsSpan(0, length);
try
{
if (valueSpan.IsEmpty)
{
valueSequence.CopyTo(rentedSpan);
}
else
{
valueSpan.CopyTo(rentedSpan);
}
document = Parse(rented.AsMemory(0, length), state.Options, rented);
return(true);
}
catch
{
// This really shouldn't happen since the document was already checked
// for consistency by Skip. But if data mutations happened just after
// the calls to Read then the copy may not be valid.
rentedSpan.Clear();
ArrayPool <byte> .Shared.Return(rented);
throw;
}
}
public Parser(ReadOnlySequence <Char> sequence)
{
lexer = new Lexer <TNumber>(new SequenceReader <Char>(sequence));
}
public void TryReadTo_SkipDelimiter()
{
byte[] expected = Encoding.UTF8.GetBytes("This is our ^|understanding^|");
ReadOnlySequence <byte> bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^|| you see.");
SequenceReader <byte> reader = new SequenceReader <byte>(bytes);
Assert.True(reader.TryReadTo(out ReadOnlySpan <byte> span, (byte)'|', (byte)'^', advancePastDelimiter: true));
Assert.Equal(expected, span.ToArray());
Assert.True(reader.IsNext((byte)' '));
Assert.Equal(30, reader.Consumed);
reader = new SequenceReader <byte>(bytes);
Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false));
Assert.Equal(expected, span.ToArray());
Assert.True(reader.IsNext((byte)'|'));
Assert.Equal(29, reader.Consumed);
// Put the skip delimiter in another segment
bytes = SequenceFactory.CreateUtf8("This is our ^|understanding", "^|| you see.");
reader = new SequenceReader <byte>(bytes);
Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true));
Assert.Equal(expected, span.ToArray());
Assert.True(reader.IsNext((byte)' '));
Assert.Equal(30, reader.Consumed);
reader = new SequenceReader <byte>(bytes);
Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false));
Assert.Equal(expected, span.ToArray());
Assert.True(reader.IsNext((byte)'|'));
Assert.Equal(29, reader.Consumed);
// Put the skip delimiter at the end of the segment
bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^", "|| you see.");
reader = new SequenceReader <byte>(bytes);
Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true));
Assert.Equal(expected, span.ToArray());
Assert.True(reader.IsNext((byte)' '));
Assert.Equal(30, reader.Consumed);
reader = new SequenceReader <byte>(bytes);
Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false));
Assert.Equal(expected, span.ToArray());
Assert.True(reader.IsNext((byte)'|'));
Assert.Equal(29, reader.Consumed);
// No trailing data
bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^||");
reader = new SequenceReader <byte>(bytes);
Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false));
Assert.Equal(expected, span.ToArray());
Assert.True(reader.IsNext((byte)'|'));
Assert.Equal(29, reader.Consumed);
reader = new SequenceReader <byte>(bytes);
Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true));
Assert.Equal(expected, span.ToArray());
Assert.True(reader.End);
Assert.Equal(30, reader.Consumed);
// All delimiters skipped
bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^|");
reader = new SequenceReader <byte>(bytes);
Assert.False(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false));
Assert.Equal(0, reader.Consumed);
reader = new SequenceReader <byte>(bytes);
Assert.False(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true));
Assert.Equal(0, reader.Consumed);
bytes = SequenceFactory.CreateUtf8("abc^|de|");
reader = new SequenceReader <byte>(bytes);
Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true));
Assert.Equal(Encoding.UTF8.GetBytes("abc^|de"), span.ToArray());
Assert.True(reader.End);
Assert.Equal(8, reader.Consumed);
// Escape leads
bytes = SequenceFactory.CreateUtf8("^|a|b");
reader = new SequenceReader <byte>(bytes);
Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true));
Assert.Equal(Encoding.UTF8.GetBytes("^|a"), span.ToArray());
Assert.True(reader.IsNext((byte)'b'));
Assert.Equal(4, reader.Consumed);
// Delimiter starts second segment.
bytes = SequenceFactory.CreateUtf8("^", "|a|b");
reader = new SequenceReader <byte>(bytes);
Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true));
Assert.Equal(Encoding.UTF8.GetBytes("^|a"), span.ToArray());
Assert.True(reader.IsNext((byte)'b'));
Assert.Equal(4, reader.Consumed);
}
public Tokenizer(ReadOnlySequence <byte> value)
{
_value = new BufferReader(value);
Current = default;
}