public ParseResult ParseMessage(ReadOnlySequence <byte> buffer, out SequencePosition consumed, out SequencePosition examined, out byte[]?message) { consumed = buffer.Start; examined = buffer.End; message = null; var start = consumed; while (buffer.Length > 0) { if (!(buffer.PositionOf(ByteLF) is SequencePosition lineEnd)) { // For the case of data: Foo\r\n\r\<Anything except \n> if (_internalParserState == InternalParseState.ReadEndOfMessage) { if (ConvertBufferToSpan(buffer.Slice(start, buffer.End)).Length > 1) { throw new FormatException("Expected a \\r\\n frame ending"); } } // Partial message. We need to read more. return(ParseResult.Incomplete); } lineEnd = buffer.GetPosition(1, lineEnd); var line = ConvertBufferToSpan(buffer.Slice(start, lineEnd)); buffer = buffer.Slice(line.Length); if (line.Length <= 1) { throw new FormatException("There was an error in the frame format"); } // Skip comments if (line[0] == ByteColon) { start = lineEnd; consumed = lineEnd; continue; } if (IsMessageEnd(line)) { _internalParserState = InternalParseState.ReadEndOfMessage; } // To ensure that the \n was preceded by a \r // since messages can't contain \n. // data: foo\n\bar should be encoded as // data: foo\r\n // data: bar\r\n else if (line[line.Length - SseLineEnding.Length] != ByteCR) { throw new FormatException("Unexpected '\\n' in message. A '\\n' character can only be used as part of the newline sequence '\\r\\n'"); } else { EnsureStartsWithDataPrefix(line); } var payload = Array.Empty <byte>(); switch (_internalParserState) { case InternalParseState.ReadMessagePayload: EnsureStartsWithDataPrefix(line); // Slice away the 'data: ' var payloadLength = line.Length - (DataPrefix.Length + SseLineEnding.Length); var newData = line.Slice(DataPrefix.Length, payloadLength).ToArray(); _data.Add(newData); start = lineEnd; consumed = lineEnd; break; case InternalParseState.ReadEndOfMessage: if (_data.Count == 1) { payload = _data[0]; } else if (_data.Count > 1) { // Find the final size of the payload var payloadSize = 0; foreach (var dataLine in _data) { payloadSize += dataLine.Length; } payloadSize += _newLine.Length * _data.Count; // Allocate space in the payload buffer for the data and the new lines. // Subtract newLine length because we don't want a trailing newline. payload = new byte[payloadSize - _newLine.Length]; var offset = 0; foreach (var dataLine in _data) { dataLine.CopyTo(payload, offset); offset += dataLine.Length; if (offset < payload.Length) { _newLine.CopyTo(payload, offset); offset += _newLine.Length; } } } message = payload; consumed = lineEnd; examined = consumed; return(ParseResult.Completed); } if (buffer.Length > 0 && buffer.First.Span[0] == ByteCR) { _internalParserState = InternalParseState.ReadEndOfMessage; } } return(ParseResult.Incomplete); }
private async ValueTask <MessageId> Send(PulsarApi.MessageMetadata metadata, ReadOnlySequence <byte> data, CancellationToken cancellationToken) { var response = await _channel.Send(metadata, data, cancellationToken).ConfigureAwait(false); return(response.MessageId.ToMessageId()); }
public void TryReadTo_SkipDelimiter_Runs() { ReadOnlySequence <byte> bytes = SequenceFactory.CreateUtf8("abc^^|def"); SequenceReader <byte> reader = new SequenceReader <byte>(bytes); Assert.True(reader.TryReadTo(out ReadOnlySpan <byte> span, (byte)'|', (byte)'^', advancePastDelimiter: false)); Assert.Equal(Encoding.UTF8.GetBytes("abc^^"), span.ToArray()); Assert.True(reader.IsNext((byte)'|')); Assert.Equal(5, reader.Consumed); // Split after escape char bytes = SequenceFactory.CreateUtf8("abc^^", "|def"); reader = new SequenceReader <byte>(bytes); Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); Assert.Equal(Encoding.UTF8.GetBytes("abc^^"), span.ToArray()); Assert.True(reader.IsNext((byte)'|')); Assert.Equal(5, reader.Consumed); // Split before and after escape char bytes = SequenceFactory.CreateUtf8("abc^", "^", "|def"); reader = new SequenceReader <byte>(bytes); Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); Assert.Equal(Encoding.UTF8.GetBytes("abc^^"), span.ToArray()); Assert.True(reader.IsNext((byte)'|')); Assert.Equal(5, reader.Consumed); // Check advance past delimiter reader = new SequenceReader <byte>(bytes); Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); Assert.Equal(Encoding.UTF8.GetBytes("abc^^"), span.ToArray()); Assert.True(reader.IsNext((byte)'d')); Assert.Equal(6, reader.Consumed); // Leading run of 2 bytes = SequenceFactory.CreateUtf8("^^|abc"); reader = new SequenceReader <byte>(bytes); Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); Assert.Equal(Encoding.UTF8.GetBytes("^^"), span.ToArray()); Assert.True(reader.IsNext((byte)'|')); Assert.Equal(2, reader.Consumed); // Leading run of 3 bytes = SequenceFactory.CreateUtf8("^^^|abc"); reader = new SequenceReader <byte>(bytes); Assert.False(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); Assert.True(reader.IsNext((byte)'^')); Assert.Equal(0, reader.Consumed); // Trailing run of 3 bytes = SequenceFactory.CreateUtf8("abc^^^|"); reader = new SequenceReader <byte>(bytes); Assert.False(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); Assert.True(reader.IsNext((byte)'a')); Assert.Equal(0, reader.Consumed); // Trailing run of 3, split bytes = SequenceFactory.CreateUtf8("abc^^^", "|"); reader = new SequenceReader <byte>(bytes); Assert.False(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); Assert.True(reader.IsNext((byte)'a')); Assert.Equal(0, reader.Consumed); }
private HttpHeader ParseHttpHeaderItems(ReadOnlySequence <byte> header) { var headerText = header.GetString(Encoding.UTF8); var reader = new StringReader(headerText); var firstLine = reader.ReadLine(); if (string.IsNullOrEmpty(firstLine)) { return(null); } var items = new NameValueCollection(); var prevKey = string.Empty; var line = string.Empty; while (!string.IsNullOrEmpty(line = reader.ReadLine())) { if (line.StartsWith(_TAB) && !string.IsNullOrEmpty(prevKey)) { var currentValue = items.Get(prevKey); items[prevKey] = currentValue + line.Trim(); continue; } int pos = line.IndexOf(_COLON); if (pos <= 0) { continue; } string key = line.Substring(0, pos); if (!string.IsNullOrEmpty(key)) { key = key.Trim(); } if (string.IsNullOrEmpty(key)) { continue; } var valueOffset = pos + 1; if (line.Length <= valueOffset) //No value in this line { continue; } var value = line.Substring(valueOffset); if (!string.IsNullOrEmpty(value) && value.StartsWith(' ') && value.Length > 1) { value = value.Substring(1); } var existingValue = items.Get(key); if (string.IsNullOrEmpty(existingValue)) { items.Add(key, value); } else { items[key] = existingValue + ", " + value; } prevKey = key; } var metaInfo = firstLine.Split(' '); if (metaInfo.Length != 3) { // invalid first line return(null); } return(new HttpHeader(metaInfo[0], metaInfo[1], metaInfo[2], items)); }
public IEnumerable <Dictionary <string, dynamic> > Parse() { using (FileStream fs = new FileStream(_fullFilePath, FileMode.Open, FileAccess.Read)) { bool canRead = true; bool isFirst = true; while (canRead) { byte[] chunk = new byte[CHUNK_SIZE]; if (isFirst) { _offset = 0; } else { _offset = 1; chunk[0] = 91; while (inBuffer != 0) { inBuffer--; chunk[_offset] = _buffer[inBuffer]; _offset++; } } var count = fs.Read(chunk, _offset, CHUNK_SIZE - _offset - 1); isFirst = false; if (count < CHUNK_SIZE - _offset - 1) { canRead = false; } for (int i = count + _offset - 1; i > 0; i--) { if (chunk[i] != 125) { _buffer[inBuffer] = chunk[i]; inBuffer++; } else { chunk[i + 1] = 93; break; } } if (_offset > 0) { for (int i = 1; i < CHUNK_SIZE; i++) { if (chunk[i] != 123) { chunk[i] = 32; } else { break; } } } ReadOnlySequence <byte> bytesSpan = new ReadOnlySequence <byte>(chunk); Utf8JsonReader jsonReader = new Utf8JsonReader(bytesSpan); Dictionary <string, dynamic> dataRows = JsonSerializer.Deserialize <Dictionary <string, dynamic> >(ref jsonReader, _serializerOptions); yield return(dataRows); } } yield return(_dataBlock); }
internal bool NeedsEncode(ReadOnlySequence <char> head) => Utils.FindNeedsEncode(head, 0, Config) != -1;
public override bool TryParseMessage(ref ReadOnlySequence <byte> buffer, out ServerlessMessage message) => ServerlessProtocol.TryParseMessage(ref buffer, out message);
public static void ReadZeroCharacters() { using var tr = new ReadOnlySequence <char>(CharData).AsTextReader(); Equal(0, tr.Read(new char[0], 0, 0)); }
protected abstract int GetBodyLengthFromHeader(ReadOnlySequence <byte> buffer);
internal override ValueTask WriteAsync(ReadOnlySequence <byte> buffers, CancellationToken cancellationToken = default) { return(WriteAsync(buffers, endStream: false, cancellationToken)); }
public bool Verify(ReadOnlySequence <byte> eventBuffer, ReadOnlySequence <byte> checksumBuffer) { throw new System.NotImplementedException(); }
public void TestFilterDifferentSingleMark() { ListenOptions channelOptions = new ListenOptions(); List <byte> data = new List <byte>(); List <byte> allData = new List <byte>(); ReadOnlySequence <byte> sequence; FilterInfo filterInfo = CreateFilterInfo(9); IFilter pipelineFilter = new FixHeaderSizePipelineFilter(channelOptions.ChannelOptions, filterInfo); BaizeSession baizeSession = new BaizeSession(Guid.NewGuid().ToString()); data.AddRange(new byte[] { (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4', (byte)'5', (byte)'6', (byte)0x02, (byte)0x00, (byte)'9', (byte)('1') }); sequence = new ReadOnlySequence <byte>(data.ToArray()); if (pipelineFilter.FilterData(baizeSession, sequence)) { string str = baizeSession.Data.GetString(Encoding.ASCII); Assert.Equal(Encoding.ASCII.GetString(data.ToArray()), str); } data.Clear(); data.AddRange(new byte[] { (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4', (byte)'5', (byte)'6', (byte)0x04, (byte)0x00, (byte)'9', (byte)('1') }); allData.AddRange(data); sequence = new ReadOnlySequence <byte>(data.ToArray()); Assert.False(pipelineFilter.FilterData(baizeSession, sequence)); data.Clear(); data.AddRange(new byte[] { (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4' }); allData.AddRange(data); sequence = new ReadOnlySequence <byte>(data.ToArray()); if (pipelineFilter.FilterData(baizeSession, sequence)) { string str = baizeSession.Data.GetString(Encoding.ASCII); string contentStr = Encoding.ASCII.GetString(allData.GetRange(0, 13).ToArray()); Assert.Equal(contentStr, str); } data.Clear(); data.AddRange(new byte[] { (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4', (byte)'5', (byte)'6', (byte)0x04 }); allData.AddRange(data); sequence = new ReadOnlySequence <byte>(data.ToArray()); Assert.False(pipelineFilter.FilterData(baizeSession, sequence)); data.Clear(); data.AddRange(new byte[] { (byte)0x00, (byte)'9', (byte)('1'), (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4' }); allData.AddRange(data); sequence = new ReadOnlySequence <byte>(data.ToArray()); if (pipelineFilter.FilterData(baizeSession, sequence)) { string str = baizeSession.Data.GetString(Encoding.ASCII); string contentStr = Encoding.ASCII.GetString(allData.GetRange(0, 13).ToArray()); Assert.Equal(contentStr, str); allData.Clear(); } data.Clear(); data.AddRange(new byte[] { (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4', (byte)'5', (byte)'6' }); allData.AddRange(data); sequence = new ReadOnlySequence <byte>(data.ToArray()); Assert.False(pipelineFilter.FilterData(baizeSession, sequence)); data.Clear(); data.AddRange(new byte[] { (byte)0x04, (byte)0x00, (byte)'9', (byte)('1'), (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4' }); allData.AddRange(data); sequence = new ReadOnlySequence <byte>(data.ToArray()); if (pipelineFilter.FilterData(baizeSession, sequence)) { string str = baizeSession.Data.GetString(Encoding.ASCII); string contentStr = Encoding.ASCII.GetString(allData.GetRange(0, 13).ToArray()); Assert.Equal(contentStr, str); allData.Clear(); } //测试包头长度在中间段 data.Clear(); data.AddRange(new byte[] { (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4', (byte)'5', (byte)'6' }); allData.AddRange(data); sequence = new ReadOnlySequence <byte>(data.ToArray()); Assert.False(pipelineFilter.FilterData(baizeSession, sequence)); data.Clear(); data.AddRange(new byte[] { (byte)0x04, (byte)0x00, (byte)'9', (byte)('1') }); allData.AddRange(data); sequence = new ReadOnlySequence <byte>(data.ToArray()); Assert.False(pipelineFilter.FilterData(baizeSession, sequence)); data.Clear(); data.AddRange(new byte[] { (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4' }); allData.AddRange(data); sequence = new ReadOnlySequence <byte>(data.ToArray()); if (pipelineFilter.FilterData(baizeSession, sequence)) { string str = baizeSession.Data.GetString(Encoding.ASCII); string contentStr = Encoding.ASCII.GetString(allData.GetRange(0, 13).ToArray()); Assert.Equal(contentStr, str); allData.Clear(); } }
public int GetBodyLength(ReadOnlySequence <byte> sequence) { int rtn = sequence.GetInt16(7); return(rtn); }
private Message DecodeMessage(FramingConnection connection, ref ReadOnlySequence <byte> buffer) { // TODO: plumb through from binding int maxBufferSize = TransportDefaults.MaxBufferSize; var decoder = connection.ServerSessionDecoder; while (!connection.EOF && buffer.Length > 0) { int bytesRead = decoder.Decode(buffer); if (bytesRead > 0) { if (!connection.EnvelopeBuffer.IsEmpty) { var remainingEnvelopeBuffer = connection.EnvelopeBuffer.Slice(connection.EnvelopeOffset, connection.EnvelopeSize - connection.EnvelopeOffset); CopyBuffer(buffer, remainingEnvelopeBuffer, bytesRead); connection.EnvelopeOffset += bytesRead; } buffer = buffer.Slice(bytesRead); } switch (decoder.CurrentState) { case ServerSessionDecoder.State.EnvelopeStart: int envelopeSize = decoder.EnvelopeSize; if (envelopeSize > maxBufferSize) { // TODO: Sending faults //base.SendFault(FramingEncodingString.MaxMessageSizeExceededFault, timeout); throw DiagnosticUtility.ExceptionUtility.ThrowHelperError( MaxMessageSizeStream.CreateMaxReceivedMessageSizeExceededException(maxBufferSize)); } connection.EnvelopeBuffer = connection.BufferManager.TakeBuffer(envelopeSize); connection.EnvelopeSize = envelopeSize; connection.EnvelopeOffset = 0; break; case ServerSessionDecoder.State.EnvelopeEnd: if (!connection.EnvelopeBuffer.IsEmpty) { Message message = null; try { message = connection.MessageEncoder.ReadMessage( new ArraySegment <byte>(connection.EnvelopeBuffer.ToArray(), 0, connection.EnvelopeSize), connection.BufferManager, connection.ServerSessionDecoder.ContentType); } catch (XmlException xmlException) { throw DiagnosticUtility.ExceptionUtility.ThrowHelperError( new ProtocolException(SR.MessageXmlProtocolError, xmlException)); } connection.EnvelopeBuffer = null; return(message); } break; case ServerSessionDecoder.State.End: connection.EOF = true; break; } } return(null); }
public async Task SocketCanReadAndWrite() { var loop = new UvLoopHandle(_logger); loop.Init(_uv); var tcp = new UvTcpHandle(_logger); tcp.Init(loop, (a, b) => { }); var endPoint = new IPEndPoint(IPAddress.Loopback, 0); tcp.Bind(endPoint); var port = tcp.GetSockIPEndPoint().Port; tcp.Listen(10, (_, status, error, state) => { var tcp2 = new UvTcpHandle(_logger); tcp2.Init(loop, (a, b) => { }); tcp.Accept(tcp2); var data = Marshal.AllocCoTaskMem(500); tcp2.ReadStart( (a, b, c) => tcp2.Libuv.buf_init(data, 500), async(__, nread, state2) => { if (nread <= 0) { tcp2.Dispose(); } else { for (var x = 0; x < 2; x++) { var req = new UvWriteReq(_logger); req.DangerousInit(loop); var block = new ReadOnlySequence <byte>(new byte[] { 65, 66, 67, 68, 69 }); await req.WriteAsync( tcp2, block); } } }, null); tcp.Dispose(); }, null); var t = Task.Run(async() => { var socket = TestConnection.CreateConnectedLoopbackSocket(port); await socket.SendAsync(new[] { new ArraySegment <byte>(new byte[] { 1, 2, 3, 4, 5 }) }, SocketFlags.None); socket.Shutdown(SocketShutdown.Send); var buffer = new ArraySegment <byte>(new byte[2048]); while (true) { var count = await socket.ReceiveAsync(new[] { buffer }, SocketFlags.None); if (count <= 0) { break; } } socket.Dispose(); }); loop.Run(); loop.Dispose(); await t; }
/// <summary> /// Initializes a new binary DTO. /// </summary> /// <param name="content">The content of the object.</param> public BinaryTransferObject(ReadOnlySequence <byte> content) => Content = content;
protected override unsafe void UploadIndices <T>(ReadOnlySequence <T> indices) { UploadBuffer(ref indexBuffer, GLEnum.ELEMENT_ARRAY_BUFFER, indices, ref indexBufferSize); }
protected override TextPackageInfo DecodePackage(ref ReadOnlySequence <byte> buffer) { return(new TextPackageInfo { Text = buffer.GetString(Utf8Encoding) }); }
public IKzWriter Add(ReadOnlySequence <byte> data) { _alg.TransformBlock(data); return(this); }
// For multi-segment parsing of a read only sequence private void ParseValuesSlow( ref ReadOnlySequence <byte> buffer, ref KeyValueAccumulator accumulator, bool isFinalBlock) { var sequenceReader = new SequenceReader <byte>(buffer); ReadOnlySequence <byte> keyValuePair; var consumed = sequenceReader.Position; var consumedBytes = default(long); var equalsDelimiter = GetEqualsForEncoding(); var andDelimiter = GetAndForEncoding(); while (!sequenceReader.End) { if (!sequenceReader.TryReadTo(out keyValuePair, andDelimiter)) { if (!isFinalBlock) { // Don't buffer indefinately if ((uint)(sequenceReader.Consumed - consumedBytes) > (uint)KeyLengthLimit + (uint)ValueLengthLimit) { ThrowKeyOrValueTooLargeException(); } break; } // This must be the final key=value pair keyValuePair = buffer.Slice(sequenceReader.Position); sequenceReader.Advance(keyValuePair.Length); } if (keyValuePair.IsSingleSegment) { ParseFormValuesFast(keyValuePair.FirstSpan, ref accumulator, isFinalBlock: true, out var segmentConsumed); Debug.Assert(segmentConsumed == keyValuePair.FirstSpan.Length); consumedBytes = sequenceReader.Consumed; consumed = sequenceReader.Position; continue; } var keyValueReader = new SequenceReader <byte>(keyValuePair); ReadOnlySequence <byte> value; if (keyValueReader.TryReadTo(out var key, equalsDelimiter)) { if (key.Length > KeyLengthLimit) { ThrowKeyTooLargeException(); } value = keyValuePair.Slice(keyValueReader.Position); if (value.Length > ValueLengthLimit) { ThrowValueTooLargeException(); } } else { // Too long for the whole segment to be a key. if (keyValuePair.Length > KeyLengthLimit) { ThrowKeyTooLargeException(); } // There is no more data, this segment must be "key" with no equals or value. key = keyValuePair; value = default; } var decodedKey = GetDecodedStringFromReadOnlySequence(key); var decodedValue = GetDecodedStringFromReadOnlySequence(value); AppendAndVerify(ref accumulator, decodedKey, decodedValue); consumedBytes = sequenceReader.Consumed; consumed = sequenceReader.Position; } buffer = buffer.Slice(consumed); }
private void InsertData(byte[] data) { _buffer = new ReadOnlySequence <byte>(data); }
internal abstract bool VerifySignature(Span <byte> data, ReadOnlySequence <byte> signature);
KzScript(ReadOnlySequence <byte> script) : this() { _script = script; }
public ReadOnlySequenceContent(ReadOnlySequence <byte> bytes) => _bytes = bytes;
public bool TryParseMessage(ref ReadOnlySequence <byte> input, IInvocationBinder binder, out HubMessage message) { throw new NotSupportedException(); }
public static HttpPipelineRequestContent Create(ReadOnlySequence <byte> bytes) => new ReadOnlySequenceContent(bytes);
private static bool TryParseValue(ref Utf8JsonReader reader, out JsonDocument document, bool shouldThrow) { JsonReaderState state = reader.CurrentState; CheckSupportedOptions(state.Options, nameof(reader)); // Value copy to overwrite the ref on an exception and undo the destructive reads. Utf8JsonReader restore = reader; ReadOnlySpan <byte> valueSpan = default; ReadOnlySequence <byte> valueSequence = default; try { switch (reader.TokenType) { // A new reader was created and has never been read, // so we need to move to the first token. // (or a reader has terminated and we're about to throw) case JsonTokenType.None: // Using a reader loop the caller has identified a property they wish to // hydrate into a JsonDocument. Move to the value first. case JsonTokenType.PropertyName: { if (!reader.Read()) { if (shouldThrow) { ThrowHelper.ThrowJsonReaderException( ref reader, ExceptionResource.ExpectedJsonTokens); } reader = restore; document = null; return(false); } break; } } switch (reader.TokenType) { // Any of the "value start" states are acceptable. case JsonTokenType.StartObject: case JsonTokenType.StartArray: { long startingOffset = reader.TokenStartIndex; // Placeholder until reader.Skip() is written (#33295) { int depth = reader.CurrentDepth; // CurrentDepth rises late and falls fast, // a payload of "[ 1, 2, 3, 4 ]" will report post-Read() // CurrentDepth values of { 0, 1, 1, 1, 1, 0 }, // Since we're logically at 0 ([), Read() once and keep // reading until we've come back down to 0 (]). do { if (!reader.Read()) { if (shouldThrow) { ThrowHelper.ThrowJsonReaderException( ref reader, ExceptionResource.ExpectedJsonTokens); } reader = restore; document = null; return(false); } } while (reader.CurrentDepth > depth); } long totalLength = reader.BytesConsumed - startingOffset; ReadOnlySequence <byte> sequence = reader.OriginalSequence; if (sequence.IsEmpty) { valueSpan = reader.OriginalSpan.Slice( checked ((int)startingOffset), checked ((int)totalLength)); } else { valueSequence = sequence.Slice(startingOffset, totalLength); } Debug.Assert( reader.TokenType == JsonTokenType.EndObject || reader.TokenType == JsonTokenType.EndArray); break; } // Single-token values case JsonTokenType.Number: case JsonTokenType.True: case JsonTokenType.False: case JsonTokenType.Null: { if (reader.HasValueSequence) { valueSequence = reader.ValueSequence; } else { valueSpan = reader.ValueSpan; } break; } // String's ValueSequence/ValueSpan omits the quotes, we need them back. case JsonTokenType.String: { ReadOnlySequence <byte> sequence = reader.OriginalSequence; if (sequence.IsEmpty) { // Since the quoted string fit in a ReadOnlySpan originally // the contents length plus the two quotes can't overflow. int payloadLength = reader.ValueSpan.Length + 2; Debug.Assert(payloadLength > 1); ReadOnlySpan <byte> readerSpan = reader.OriginalSpan; Debug.Assert( readerSpan[(int)reader.TokenStartIndex] == (byte)'"', $"Calculated span starts with {readerSpan[(int)reader.TokenStartIndex]}"); Debug.Assert( readerSpan[(int)reader.TokenStartIndex + payloadLength - 1] == (byte)'"', $"Calculated span ends with {readerSpan[(int)reader.TokenStartIndex + payloadLength - 1]}"); valueSpan = readerSpan.Slice((int)reader.TokenStartIndex, payloadLength); } else { long payloadLength = 2; if (reader.HasValueSequence) { payloadLength += reader.ValueSequence.Length; } else { payloadLength += reader.ValueSpan.Length; } valueSequence = sequence.Slice(reader.TokenStartIndex, payloadLength); Debug.Assert( valueSequence.First.Span[0] == (byte)'"', $"Calculated sequence starts with {valueSequence.First.Span[0]}"); Debug.Assert( valueSequence.ToArray()[payloadLength - 1] == (byte)'"', $"Calculated sequence ends with {valueSequence.ToArray()[payloadLength - 1]}"); } break; } default: { if (shouldThrow) { byte displayByte; if (reader.HasValueSequence) { displayByte = reader.ValueSequence.First.Span[0]; } else { displayByte = reader.ValueSpan[0]; } ThrowHelper.ThrowJsonReaderException( ref reader, ExceptionResource.ExpectedStartOfValueNotFound, displayByte); } reader = restore; document = null; return(false); } } } catch { reader = restore; throw; } int length = valueSpan.IsEmpty ? checked ((int)valueSequence.Length) : valueSpan.Length; byte[] rented = ArrayPool <byte> .Shared.Rent(length); Span <byte> rentedSpan = rented.AsSpan(0, length); try { if (valueSpan.IsEmpty) { valueSequence.CopyTo(rentedSpan); } else { valueSpan.CopyTo(rentedSpan); } document = Parse(rented.AsMemory(0, length), state.Options, rented); return(true); } catch { // This really shouldn't happen since the document was already checked // for consistency by Skip. But if data mutations happened just after // the calls to Read then the copy may not be valid. rentedSpan.Clear(); ArrayPool <byte> .Shared.Return(rented); throw; } }
public Parser(ReadOnlySequence <Char> sequence) { lexer = new Lexer <TNumber>(new SequenceReader <Char>(sequence)); }
public void TryReadTo_SkipDelimiter() { byte[] expected = Encoding.UTF8.GetBytes("This is our ^|understanding^|"); ReadOnlySequence <byte> bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^|| you see."); SequenceReader <byte> reader = new SequenceReader <byte>(bytes); Assert.True(reader.TryReadTo(out ReadOnlySpan <byte> span, (byte)'|', (byte)'^', advancePastDelimiter: true)); Assert.Equal(expected, span.ToArray()); Assert.True(reader.IsNext((byte)' ')); Assert.Equal(30, reader.Consumed); reader = new SequenceReader <byte>(bytes); Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); Assert.Equal(expected, span.ToArray()); Assert.True(reader.IsNext((byte)'|')); Assert.Equal(29, reader.Consumed); // Put the skip delimiter in another segment bytes = SequenceFactory.CreateUtf8("This is our ^|understanding", "^|| you see."); reader = new SequenceReader <byte>(bytes); Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); Assert.Equal(expected, span.ToArray()); Assert.True(reader.IsNext((byte)' ')); Assert.Equal(30, reader.Consumed); reader = new SequenceReader <byte>(bytes); Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); Assert.Equal(expected, span.ToArray()); Assert.True(reader.IsNext((byte)'|')); Assert.Equal(29, reader.Consumed); // Put the skip delimiter at the end of the segment bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^", "|| you see."); reader = new SequenceReader <byte>(bytes); Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); Assert.Equal(expected, span.ToArray()); Assert.True(reader.IsNext((byte)' ')); Assert.Equal(30, reader.Consumed); reader = new SequenceReader <byte>(bytes); Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); Assert.Equal(expected, span.ToArray()); Assert.True(reader.IsNext((byte)'|')); Assert.Equal(29, reader.Consumed); // No trailing data bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^||"); reader = new SequenceReader <byte>(bytes); Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); Assert.Equal(expected, span.ToArray()); Assert.True(reader.IsNext((byte)'|')); Assert.Equal(29, reader.Consumed); reader = new SequenceReader <byte>(bytes); Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); Assert.Equal(expected, span.ToArray()); Assert.True(reader.End); Assert.Equal(30, reader.Consumed); // All delimiters skipped bytes = SequenceFactory.CreateUtf8("This is our ^|understanding^|"); reader = new SequenceReader <byte>(bytes); Assert.False(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: false)); Assert.Equal(0, reader.Consumed); reader = new SequenceReader <byte>(bytes); Assert.False(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); Assert.Equal(0, reader.Consumed); bytes = SequenceFactory.CreateUtf8("abc^|de|"); reader = new SequenceReader <byte>(bytes); Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); Assert.Equal(Encoding.UTF8.GetBytes("abc^|de"), span.ToArray()); Assert.True(reader.End); Assert.Equal(8, reader.Consumed); // Escape leads bytes = SequenceFactory.CreateUtf8("^|a|b"); reader = new SequenceReader <byte>(bytes); Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); Assert.Equal(Encoding.UTF8.GetBytes("^|a"), span.ToArray()); Assert.True(reader.IsNext((byte)'b')); Assert.Equal(4, reader.Consumed); // Delimiter starts second segment. bytes = SequenceFactory.CreateUtf8("^", "|a|b"); reader = new SequenceReader <byte>(bytes); Assert.True(reader.TryReadTo(out span, (byte)'|', (byte)'^', advancePastDelimiter: true)); Assert.Equal(Encoding.UTF8.GetBytes("^|a"), span.ToArray()); Assert.True(reader.IsNext((byte)'b')); Assert.Equal(4, reader.Consumed); }
public Tokenizer(ReadOnlySequence <byte> value) { _value = new BufferReader(value); Current = default; }