private static bool TryParseEvent(ReadOnlySequence <byte> buffer, out Event nextEvent, out SequencePosition consumedTo) { // Find the start of the event var begin = buffer.PositionOf(UTF8Constants.BeginEvent.Span); if (begin == null) { nextEvent = default; consumedTo = buffer.Start; return(false); } // We found a beginning. Throw away data up to that point. consumedTo = begin.Value; // Can we also find an end to it? var end = buffer.PositionOf(UTF8Constants.EndEvent.Span); if (end == null) { nextEvent = default; consumedTo = default; return(false); } var payload = buffer.Slice(buffer.GetPosition(UTF8Constants.BeginEvent.Length, begin.Value), end.Value); ReadEvent(payload, out nextEvent); consumedTo = buffer.GetPosition(UTF8Constants.EndEvent.Length, end.Value); return(true); }
private static bool TryParseFrame(ReadOnlySequence <byte> buffer, out string nextMessage, out SequencePosition consumedTo) { // "Content-Length: \r\n\r\n".Length; const int PREFIX_LEN = 20; // find the end-of-line marker var eol = buffer.PositionOf((byte)'\r'); if (eol == null) { nextMessage = default; consumedTo = default; return(false); } // Parse the length string // "Content-Length: {len}\r\n\r\n" var lengthSequence = buffer.Slice(buffer.PositionOf((byte)' ').Value, eol.Value); Span <byte> lengthBytes = stackalloc byte[(int)lengthSequence.Length]; lengthSequence.CopyTo(lengthBytes); var lengthInt = int.Parse(Encoding.UTF8.GetString(lengthBytes), CultureInfo.InvariantCulture); // TODO: fix fast utf8->uint parsing //if (!Utf8Parser.TryParse(lengthBytes, out uint xlengthInt, out var lengthParseBytesConsumed, standardFormat: 'D')) //{ // throw new Exception("failed"); //} int fullPrefixLength = PREFIX_LEN + (int)lengthSequence.Length; int fullMessageLength = fullPrefixLength + lengthInt; if (buffer.Length < fullMessageLength) { nextMessage = default; consumedTo = default; return(false); } // read past the message consumedTo = buffer.GetPosition(fullMessageLength - 1); // consume the data var payload = buffer.Slice(fullPrefixLength - 1, lengthInt); nextMessage = GetString(payload); return(true); }
private string ReadLine(ReadOnlySequence <byte> buffer, out SequencePosition consumed) { consumed = buffer.Start; var rindex = buffer.PositionOf((byte)'\r'); var nindex = buffer.PositionOf((byte)'\n'); if (rindex.HasValue && nindex.HasValue) { var lineSequence = buffer.Slice(consumed, nindex.Value); var linebytes = lineSequence.Slice(0, lineSequence.Length - 1).ToArray(); consumed = buffer.GetPosition(linebytes.Length + 2); return(defaultEncoding.GetString(linebytes)); } return(null); }
public DataSegment Deserialize(ReadOnlySequence <byte> buffer) { // length SequencePosition?pos = buffer.PositionOf((byte)','); if (pos == null) { throw new DataMismatchException("should contain comma-separated values"); } ReadOnlySequence <byte> lengthBytes = buffer.Slice(0, pos.Value); int numSize = s_encoding.GetChars(lengthBytes.ToArray(), _charBuffer); int length = int.Parse(_charBuffer.AsSpan().Slice(0, numSize)); buffer = buffer.Slice(buffer.GetPosition(1, pos.Value)); // checksum pos = buffer.PositionOf((byte)','); if (pos == null) { throw new DataMismatchException("should contain comma-separated values"); } ReadOnlySequence <byte> checksumBytes = buffer.Slice(0, pos.Value); numSize = s_encoding.GetChars(checksumBytes.ToArray(), _charBuffer); ulong checksum = ulong.Parse(_charBuffer.AsSpan().Slice(0, numSize)); buffer = buffer.Slice(buffer.GetPosition(1, pos.Value)); // payload if (length != (int)buffer.Length) { throw new DataMismatchException("declared length does not match payload length"); } var chunk = new DataSegment((int)buffer.Length); buffer.CopyTo(chunk.AsSpan()); if (checksum != chunk.Checksum) { chunk.Return(); throw new DataMismatchException("declared checksum doesn't match payload checksum"); } return(chunk); }
static async Task ReadFromPipe(PipeReader reader, Action <ReadOnlySequence <byte> > operation) { while (true) { var readResult = await reader.ReadAsync(); ReadOnlySequence <byte> buffer = readResult.Buffer; SequencePosition? position = null; do { position = buffer.PositionOf((byte)'\n'); if (position != null) { var line = buffer.Slice(0, position.Value); operation?.Invoke(line); var next = buffer.GetPosition(1, position.Value); //等价于popstion+1因为需要包含'\n'; buffer = buffer.Slice(next); //跳过已经读取的数据; } }while (position != null); //指示PipeReader已经消费了多少数据 reader.AdvanceTo(buffer.Start, buffer.End); if (readResult.IsCompleted) { break; } } reader.Complete(); }
/// <summary> /// /// </summary> /// <param name="cancellationToken"></param> /// <returns></returns> public async Task ReadFromPipeAsync(System.Threading.CancellationToken cancellationToken = default) { while (true) { try { ReadResult result = await _payloadPipe.Reader.ReadAsync(); ReadOnlySequence <byte> resultBuffer = result.Buffer; SequencePosition? pos; do { pos = resultBuffer.PositionOf((byte)'\0'); if (pos != null) { await ProcessPayload(resultBuffer.Slice(0, pos.Value), cancellationToken); resultBuffer = resultBuffer.Slice(resultBuffer.GetPosition(1, pos.Value)); } } while (pos != null); _payloadPipe.Reader.AdvanceTo(resultBuffer.Start, resultBuffer.End); if (result.IsCompleted) { break; } } catch (Exception ex) { _logger.LogError(ex, "Error occurred while processing data from pipe."); } } await _payloadPipe.Reader.CompleteAsync(); }
public void TestSeekIteratorLimitWithinSameBlock(string input, char seek, char limitAfter, int expectedReturnValue) { ReadOnlySequence <byte> originalBuffer = Factory.CreateWithContent(input); SequencePosition scan1 = originalBuffer.Start; ReadOnlySequence <byte> buffer = originalBuffer; // Act SequencePosition?end = originalBuffer.PositionOf((byte)limitAfter); if (end.HasValue) { buffer = originalBuffer.Slice(buffer.Start, buffer.GetPosition(end.Value, 1)); } SequencePosition?returnValue1 = buffer.PositionOf((byte)seek); // Assert Assert.Equal(input.Contains(limitAfter), end.HasValue); if (expectedReturnValue != -1) { int expectedEndIndex = input.IndexOf(seek); Assert.NotNull(returnValue1); Assert.Equal(Encoding.ASCII.GetBytes(input.Substring(expectedEndIndex)), originalBuffer.Slice(returnValue1.Value).ToArray()); } }
public void TestSeekByteLimitWithinSameBlock(string input, char seek, int limit, int expectedBytesScanned, int expectedReturnValue) { // Arrange ReadOnlySequence <byte> originalBuffer = Factory.CreateWithContent(input); // Act ReadOnlySequence <byte> buffer = limit > input.Length ? originalBuffer : originalBuffer.Slice(0, limit); SequencePosition?result = buffer.PositionOf((byte)seek); // Assert if (expectedReturnValue == -1) { Assert.Null(result); } else { Assert.NotNull(result); } if (expectedReturnValue != -1) { Assert.Equal(Encoding.ASCII.GetBytes(input.Substring(expectedBytesScanned - 1)), originalBuffer.Slice(result.Value).ToArray()); } }
private static int CalculateRecordLength(ReadOnlySequence <byte> span) { // need to figure out how long the header is //var position = span.PositionOf((byte) '\n'); //long lfPos; //if (position == null) //{ // lfPos = span.Length - 1; //} //else //{ // lfPos = position.Value.GetInteger(); //} //while (nextValuePos ) //{ // headerCount++; // var nextPos = span.Slice(nextValuePos).PositionOf((byte)','); // var length = nextPos?.GetInteger() + 1 ?? 0; // nextValuePos += length; // if (length == 0) break; // case for single line csv //} int headerCount = 1; SequencePosition?nextPostion = span.PositionOf((byte)','); while (nextPostion != null) { headerCount++; nextPostion = span.Slice(span.GetPosition(1, nextPostion.Value)).PositionOf((byte)','); } return(headerCount); }
protected virtual async Task ProcessCommandLine(Guid clientId, ReadOnlySequence <byte> line) { SequencePosition?position = line.PositionOf((byte)' '); var command = line.Slice(0, position.Value); var data = line.Slice(line.GetPosition(1, position.Value)); //Build command var stringBuilder = new StringBuilder(); foreach (var segment in command) { stringBuilder.Append(Encoding.ASCII.GetString(segment.Span).ToLower()); } var parsedCommand = stringBuilder.ToString(); //Build data stringBuilder.Clear(); foreach (var segment in data) { stringBuilder.Append(Encoding.ASCII.GetString(segment.Span)); } var parsedData = stringBuilder.ToString(); await ProcessMessage(new Message { Service = Enum.Parse(typeof(Service), parsedCommand, true) as Service?, Data = parsedData, ClientId = clientId }); }
public static async Task WriteMessageAsync(ReadOnlySequence <byte> payload, Stream output) { // Payload does not contain a line feed so write it directly to output if (payload.PositionOf(LineFeed) == null) { if (payload.Length > 0) { await output.WriteAsync(DataPrefix, 0, DataPrefix.Length); await output.WriteAsync(payload); await output.WriteAsync(Newline, 0, Newline.Length); } await output.WriteAsync(Newline, 0, Newline.Length); return; } var ms = new MemoryStream(); // Parse payload and write formatted output to memory await WriteMessageToMemory(ms, payload); ms.Position = 0; await ms.CopyToAsync(output); }
private static IEnumerable <(SequencePosition pos, MPEGTS.Packet packet)> ReadPackets(ReadOnlySequence <byte> buf, CancellationToken cancellationToken = default) { SequencePosition?sp = null; do { sp = buf.PositionOf(MPEGTS.SyncMarker); if (sp != null && buf.Length >= MPEGTS.PacketLength) { //try to parse a packet starting at sp var packetData = buf.Slice(sp.Value, MPEGTS.PacketLength); var pkt = MPEGTS.Packet.Parse(packetData); pkt.OriginalData = packetData; yield return(sp.Value, pkt); var nextMsgPos = buf.GetPosition(MPEGTS.PacketLength, sp.Value); buf = buf.Slice(nextMsgPos); } else { break; } } while (sp != null && !cancellationToken.IsCancellationRequested); }
protected override (SegmentationStatus status, ReadOnlySequence <byte>?segment) Read(ReadOnlySequence <byte> buffer) { var startOfSegment = Starts.Select(start => buffer.PositionOf(start)).FirstOrDefault(start => start != null); if (startOfSegment != null) { var segment = buffer.Slice(startOfSegment.Value); var endOfSegment = segment.PositionOf(End); if (endOfSegment != null) { var completeSegment = segment.Slice(0, segment.GetPosition(1, endOfSegment.Value)); if (this.Options.HasFlag(SegmentionOptions.SecondStartInvalid)) { var secondStart = Starts.Select(start => completeSegment.PositionOf(start)).FirstOrDefault(start => start != null); if (secondStart != null) { // Second start detected return(SegmentationStatus.Invalid, buffer.Slice(0, secondStart.Value)); } } return(SegmentationStatus.Complete, completeSegment); } else if (this.MaxLength.HasValue && buffer.Length > this.MaxLength) { var leftover = buffer.Length % this.MaxLength.Value; buffer = buffer.Slice(0, buffer.GetPosition(-leftover, buffer.End)); return(SegmentationStatus.Invalid, buffer); } } return(SegmentationStatus.Incomplete, null); }
public void PositionOf_ReturnsNullIfNotFound() { ReadOnlySequence <byte> buffer = Factory.CreateWithContent(new byte[] { 1, 2, 3 }); SequencePosition? result = buffer.PositionOf((byte)4); Assert.Null(result); }
public void Handle(ref SocketPipelineContext context, ref ReadOnlySequence <byte> data) { using (MemoryStream memoryStream = new MemoryStream()) { using (ICryptoTransform decryptor = this.Crypt.CreateDecryptor()) { using CryptoStream stream = new CryptoStream(memoryStream, decryptor, CryptoStreamMode.Write, leaveOpen: true); byte[] array = data.ToArray(); Base64.DecodeFromUtf8InPlace(array, out int writtenBytes); stream.Write(array, 0, writtenBytes); } ReadOnlySequence <byte> plank = new ReadOnlySequence <byte>(memoryStream.GetBuffer(), 0, (int)memoryStream.Length); //Encryption leaves some ugly null bytes SequencePosition?uglyNulls = plank.PositionOf((byte)'\0'); if (uglyNulls != null) { plank = plank.Slice(start: 0, end: uglyNulls.Value); } context.ProgressHandlerIn(ref plank); } }
private static bool TryParseLine(ReadOnlySequence <byte> buffer, Event nextEvent) { // Per RFC 5545 contentlines have the following syntax: // contentline = name *(";" param ) ":" value CRLF // meaning we'll read until ; or : and treat accordingly var valueDelim = buffer.PositionOf(UTF8Constants.Colon); if (valueDelim == null) { // The line is somehow invalid. Abort. return(false); } var nameAndParams = buffer.Slice(0, valueDelim.Value); var value = buffer.Slice(valueDelim.Value).Slice(1); // Check for parameters - for our use, we don't care about their values, so we simply ignore them. var paramDelim = nameAndParams.PositionOf(UTF8Constants.Semicolon); var name = paramDelim == null ? nameAndParams : nameAndParams.Slice(0, paramDelim.Value); var parameters = paramDelim == null ? new ReadOnlySequence <byte>() : nameAndParams.Slice(paramDelim.Value).Slice(1); UpdateProperty(name, parameters, value, nextEvent); return(true); }
static bool TryParseLines( ref ReadOnlySequence <byte> buffer, out string message) { SequencePosition?position; StringBuilder outputMessage = new(); while (true) { position = buffer.PositionOf((byte)'\n'); if (!position.HasValue) { break; } outputMessage.Append(Encoding.ASCII.GetString(buffer.Slice(buffer.Start, position.Value))) .AppendLine(); buffer = buffer.Slice(buffer.GetPosition(1, position.Value)); } ; message = outputMessage.ToString(); return(message.Length != 0); }
public object ManualSplitMemoryT() { var sourceMem = Source.AsMemory(); var readTotal = 0; var len = sourceMem.Length * 4; Span <byte> mem = stackalloc byte[len]; //using (var mem = MemoryPool<byte>.Shared.Rent(len)) //{ var seq = new ReadOnlySequence <char>(sourceMem); SequencePosition?pos = null; while ((pos = seq.PositionOf(CharDot)) != null) { var part = seq.Slice(seq.Start, pos.Value); seq = seq.Slice(seq.GetPosition(1, pos.Value)); // Label memory. var slice = mem.Slice(readTotal); // Write label. var read = Encoding.UTF8.GetBytes(part.First.Span, slice.Slice(1)); // Write label length prefix. slice[0] = (byte)read; readTotal += read + 1; } return(readTotal); //} }
public void PositionOf_ReturnsNullIfNotFound() { ReadOnlySequence <char> buffer = Factory.CreateWithContent(new char[] { (char)1, (char)2, (char)3 }); SequencePosition? result = buffer.PositionOf((char)4); Assert.Null(result); }
public bool CanParse(ReadOnlySequence <byte> buffer) { bool startFound = false; bool endFound = false; if (_settings.StartDelimiter != null && _settings.StartDelimiter.Length > 0) { for (int i = 0; i < _settings.StartDelimiter.Length; i++) { start = buffer.PositionOf(_settings.StartDelimiter[i]); if (!start.HasValue) { start = null; break; } } if (start.HasValue) { buffer = buffer.Slice(start.Value); startFound = true; } } else { startFound = true; } if (_settings.EndDelimiter != null && _settings.EndDelimiter.Length > 0) { end = buffer.PositionOf(_settings.EndDelimiter[0]); endFound = end.HasValue; if (endFound) { for (int i = 1; i < this._settings.EndDelimiter.Length; i++) { var auxEnd = buffer.PositionOf(_settings.EndDelimiter[i]); if (!auxEnd.HasValue) { auxEnd = null; endFound = false; break; } } } } return(startFound & endFound); }
public void PositionOf_ReturnsPosition(string raw, char searchFor, int expectIndex) { ReadOnlySequence <byte> buffer = Factory.CreateWithContent(raw); SequencePosition? result = buffer.PositionOf((byte)searchFor); Assert.NotNull(result); Assert.Equal(buffer.Slice(result.Value).ToArray(), Encoding.ASCII.GetBytes(raw.Substring(expectIndex))); }
public void PositionOf_ReturnsPosition(string raw, char searchFor, int expectIndex) { ReadOnlySequence <char> buffer = Factory.CreateWithContent(raw.ToCharArray()); SequencePosition? result = buffer.PositionOf((char)searchFor); Assert.NotNull(result); Assert.Equal(buffer.Slice(result.Value).ToArray(), raw.Substring(expectIndex)); }
public void MemorySeek(string raw, char searchFor, int expectIndex) { ReadOnlySequence <byte> cursors = Factory.CreateWithContent(raw); SequencePosition? result = cursors.PositionOf((byte)searchFor); Assert.NotNull(result); Assert.Equal(cursors.Slice(result.Value).ToArray(), Encoding.ASCII.GetBytes(raw.Substring(expectIndex))); }
// Adapted from https://devblogs.microsoft.com/dotnet/system-io-pipelines-high-performance-io-in-net/ public static async Task ReadLinesUsingPipesAsync(this Stream stream, Func <ReadOnlySequence <byte>, Task> callback, CancellationToken token = default, char separator = '\n') { var pipe = new Pipe(); try { await StressTaskExtensions.WhenAllThrowOnFirstException(token, FillPipeAsync, ReadPipeAsync); } catch (OperationCanceledException) when(token.IsCancellationRequested) { } async Task FillPipeAsync(CancellationToken token) { try { await stream.CopyToAsync(pipe.Writer, token); } catch (Exception e) { pipe.Writer.Complete(e); throw; } pipe.Writer.Complete(); } async Task ReadPipeAsync(CancellationToken token) { while (!token.IsCancellationRequested) { ReadResult result = await pipe.Reader.ReadAsync(token); ReadOnlySequence <byte> buffer = result.Buffer; SequencePosition? position; do { position = buffer.PositionOf((byte)separator); if (position != null) { await callback(buffer.Slice(0, position.Value)); buffer = buffer.Slice(buffer.GetPosition(1, position.Value)); } }while (position != null); pipe.Reader.AdvanceTo(buffer.Start, buffer.End); if (result.IsCompleted) { break; } } } }
private bool TryReadLine(ref ReadOnlySequence <byte> buffer, out ReadOnlySequence <byte> line) { SequencePosition?CRPosition = buffer.PositionOf((byte)'\r'); SequencePosition?LFPosition = buffer.PositionOf((byte)'\n'); if (!CRPosition.HasValue && !LFPosition.HasValue) { line = default; return(false); } SequencePosition newLinePosition = GetNewLinePosition(buffer, CRPosition, LFPosition); line = buffer.Slice(0, newLinePosition); buffer = buffer.Slice(buffer.GetPosition(1, newLinePosition)); return(true); }
protected override (SegmentationStatus status, ReadOnlySequence <byte>?segment) Read(ReadOnlySequence <byte> buffer) { var startOfSegment = buffer.PositionOf(Start); if (startOfSegment != null) { var segment = buffer.Slice(startOfSegment.Value); if (segment.Length >= FixedLength) { var completeSegment = segment.Slice(0, buffer.GetPosition(FixedLength, startOfSegment.Value)); if (this.Options.HasFlag(SegmentionOptions.SecondStartInvalid)) { var secondStart = completeSegment.PositionOf(Start); if (secondStart != null) { // Second start detected return(SegmentationStatus.Invalid, buffer.Slice(0, secondStart.Value)); } } if (ExtensionDefinition != null) { var valueData = completeSegment.Slice(ExtensionDefinition.Postion, ExtensionDefinition.Length); //TODO, drop the endian check... only support little and convert var set = this.ExtensionDefinition.Endianness == Endianness.Little ? valueData.ToArray() : valueData.ToArray().Reverse().ToArray(); ulong extendedLength = 0; for (var i = 0; i < this.ExtensionDefinition.Length; i++) { extendedLength |= (ulong)set[i] << (8 * i); } var actualLength = FixedLength + (long)extendedLength; if (segment.Length < actualLength) { return(SegmentationStatus.Incomplete, buffer); } completeSegment = segment.Slice(0, buffer.GetPosition(actualLength, startOfSegment.Value)); } return(SegmentationStatus.Complete, completeSegment); } } else if (buffer.Length > this.FixedLength) { var leftover = buffer.Length % this.FixedLength; buffer = buffer.Slice(0, buffer.GetPosition(-leftover, buffer.End)); return(SegmentationStatus.Invalid, buffer); } return(SegmentationStatus.Incomplete, buffer); }
private ReadOnlySequence <byte>?GetData(ReadOnlySequence <byte> line) { var position = line.PositionOf(StartByte); if (position is null) { return(null); } return(line.Slice(position.Value)); }
internal static SequencePosition?GetEndOfLinePosition(this ReadOnlySequence <byte> buffer) { var endOfLine = buffer.PositionOf((byte)'\n'); if (endOfLine == null) { return(null); } return(buffer.GetPosition(1, endOfLine.Value)); }
internal static Message DeserializeMessage(ReadOnlySequence <byte> buffer) { var data = buffer.Slice(0, 1).ToArray(); var isSystem = (data[0] & 1) > 0; var isPrivate = (data[0] & 2) > 0; var position = buffer.PositionOf(Separator); var sender = Encoding.UTF8.GetString(buffer.Slice(1, position.Value).ToArray()); buffer = buffer.Slice(buffer.GetPosition(1, position.Value)); position = buffer.PositionOf(Separator); var receiver = Encoding.UTF8.GetString(buffer.Slice(0, position.Value).ToArray()); buffer = buffer.Slice(buffer.GetPosition(1, position.Value)); var text = Encoding.UTF8.GetString(buffer.ToArray()); return(new Message(text, sender, receiver, isSystem, isPrivate)); }
private void ParseExtension(ReadOnlySequence <byte> buffer, out SequencePosition consumed, out SequencePosition examined) { // Chunk-extensions not currently parsed // Just drain the data consumed = buffer.Start; examined = buffer.Start; do { SequencePosition?extensionCursorPosition = buffer.PositionOf(ByteCR); if (extensionCursorPosition == null) { // End marker not found yet consumed = buffer.End; examined = buffer.End; AddAndCheckConsumedBytes(buffer.Length); return; } ; var extensionCursor = extensionCursorPosition.Value; var charsToByteCRExclusive = buffer.Slice(0, extensionCursor).Length; var suffixBuffer = buffer.Slice(extensionCursor); if (suffixBuffer.Length < 2) { consumed = extensionCursor; examined = buffer.End; AddAndCheckConsumedBytes(charsToByteCRExclusive); return; } suffixBuffer = suffixBuffer.Slice(0, 2); var suffixSpan = suffixBuffer.ToSpan(); if (suffixSpan[1] == '\n') { // We consumed the \r\n at the end of the extension, so switch modes. _mode = _inputLength > 0 ? Mode.Data : Mode.Trailer; consumed = suffixBuffer.End; examined = suffixBuffer.End; AddAndCheckConsumedBytes(charsToByteCRExclusive + 2); } else { // Don't consume suffixSpan[1] in case it is also a \r. buffer = buffer.Slice(charsToByteCRExclusive + 1); consumed = extensionCursor; AddAndCheckConsumedBytes(charsToByteCRExclusive + 1); } } while (_mode == Mode.Extension); }