private int IterateGetPosition(ReadOnlySequence <T> sequence)
{
int consume = 0;
SequencePosition position = sequence.Start;
int offset = (int)(sequence.Length / 10);
SequencePosition end = sequence.GetPosition(0, sequence.End);
while (!position.Equals(end))
{
position = sequence.GetPosition(offset, position);
consume += position.GetInteger();
}
return(consume);
}
private bool ParsePreface(ReadOnlySequence <byte> readableBuffer, out SequencePosition consumed, out SequencePosition examined)
{
consumed = readableBuffer.Start;
examined = readableBuffer.End;
if (readableBuffer.Length < ClientPreface.Length)
{
return(false);
}
var span = readableBuffer.IsSingleSegment
? readableBuffer.First.Span
: readableBuffer.ToSpan();
for (var i = 0; i < ClientPreface.Length; i++)
{
if (ClientPreface[i] != span[i])
{
throw new Exception("Invalid HTTP/2 connection preface.");
}
}
consumed = examined = readableBuffer.GetPosition(ClientPreface.Length);
return(true);
}
public async Task AdvanceWithGetPositionCrossingIntoWriteHeadWorks()
{
// Create two blocks
Memory <byte> memory = _pipe.Writer.GetMemory(1);
_pipe.Writer.Advance(memory.Length);
memory = _pipe.Writer.GetMemory(1);
_pipe.Writer.Advance(memory.Length);
await _pipe.Writer.FlushAsync();
// Read single block
ReadResult readResult = await _pipe.Reader.ReadAsync();
// Allocate more memory
memory = _pipe.Writer.GetMemory(1);
// Create position that would cross into write head
ReadOnlySequence <byte> buffer = readResult.Buffer;
SequencePosition position = buffer.GetPosition(buffer.Start, buffer.Length);
// Return everything
_pipe.Reader.AdvanceTo(position);
// Advance writer
_pipe.Writer.Advance(memory.Length);
_pipe.Writer.Commit();
}
public static bool ReadFrame(ReadOnlySequence <byte> readableBuffer, Http2Frame frame, uint maxFrameSize, out SequencePosition consumed, out SequencePosition examined)
{
consumed = readableBuffer.Start;
examined = readableBuffer.End;
if (readableBuffer.Length < Http2Frame.HeaderLength)
{
return(false);
}
var headerSlice = readableBuffer.Slice(0, Http2Frame.HeaderLength);
headerSlice.CopyTo(frame.Raw);
var payloadLength = frame.PayloadLength;
if (payloadLength > maxFrameSize)
{
throw new Http2ConnectionErrorException(CoreStrings.FormatHttp2ErrorFrameOverLimit(payloadLength, maxFrameSize), Http2ErrorCode.FRAME_SIZE_ERROR);
}
var frameLength = Http2Frame.HeaderLength + payloadLength;
if (readableBuffer.Length < frameLength)
{
return(false);
}
readableBuffer.Slice(Http2Frame.HeaderLength, payloadLength).CopyTo(frame.Payload);
consumed = examined = readableBuffer.GetPosition(frameLength);
return(true);
}
private async Task ReadPipeAsync(PipeReader reader, CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
var result = await reader.ReadAsync(cancellationToken);
ReadOnlySequence <byte> buffer = result.Buffer;
bool canParse = false;
do
{
canParse = _parser.CanParse(buffer);
if (canParse)
{
ParseResult messageParsed = await _parser.ParseAsync(buffer);
_whenMessageParsed.OnNext(messageParsed.CleanMessage);
buffer = buffer.Slice(buffer.GetPosition(1, messageParsed.EndPosition));
}
}while (canParse);
reader.AdvanceTo(buffer.Start, buffer.End);
if (result.IsCompleted)
{
break;
}
}
reader.Complete();
}
public int Decode(ReadOnlySequence <byte> buffer)
{
DecoderHelper.ValidateSize(buffer.Length);
if (isValueDecoded)
{
throw DiagnosticUtility.ExceptionUtility.ThrowHelperError(new InvalidOperationException(SR.FramingValueNotAvailable));
}
int bytesConsumed = 0;
while (bytesConsumed < buffer.Length)
{
var data = buffer.First.Span;
int next = data[0];
value |= (next & 0x7F) << (index * 7);
bytesConsumed++;
if (index == LastIndex && (next & 0xF8) != 0)
{
throw DiagnosticUtility.ExceptionUtility.ThrowHelperError(new InvalidDataException(SR.FramingSizeTooLarge));
}
index++;
if ((next & 0x80) == 0)
{
isValueDecoded = true;
break;
}
buffer = buffer.Slice(buffer.GetPosition(1));
}
return(bytesConsumed);
}
static async Task ReadFromPipe(PipeReader reader, Action <ReadOnlySequence <byte> > operation)
{
while (true)
{
var readResult = await reader.ReadAsync();
ReadOnlySequence <byte> buffer = readResult.Buffer;
SequencePosition? position = null;
do
{
position = buffer.PositionOf((byte)'\n');
if (position != null)
{
var line = buffer.Slice(0, position.Value);
operation?.Invoke(line);
var next = buffer.GetPosition(1, position.Value); //等价于popstion+1因为需要包含'\n';
buffer = buffer.Slice(next); //跳过已经读取的数据；
}
}while (position != null);
//指示PipeReader已经消费了多少数据
reader.AdvanceTo(buffer.Start, buffer.End);
if (readResult.IsCompleted)
{
break;
}
}
reader.Complete();
}
protected override (SegmentationStatus status, ReadOnlySequence <byte>?segment) Read(ReadOnlySequence <byte> buffer)
{
var startOfSegment = buffer.PositionOf(Start);
if (startOfSegment != null)
{
var segment = buffer.Slice(startOfSegment.Value);
var endOfSegment = segment.PositionOf(End);
if (endOfSegment != null)
{
var completeSegment = segment.Slice(0, segment.GetPosition(1, endOfSegment.Value));
if (this.Options.HasFlag(SegmentionOptions.SecondStartInvalid))
{
var secondStart = completeSegment.PositionOf(Start);
if (secondStart != null)
{
// Second start detected
return(SegmentationStatus.Invalid, buffer.Slice(0, secondStart.Value));
}
}
return(SegmentationStatus.Complete, completeSegment);
}
else if (this.MaxLength.HasValue && buffer.Length > this.MaxLength)
{
var leftover = buffer.Length % this.MaxLength.Value;
buffer = buffer.Slice(0, buffer.GetPosition(-leftover, buffer.End));
return(SegmentationStatus.Invalid, buffer);
}
}
return(SegmentationStatus.Incomplete, null);
}
static async Task ReadPipeAsync(IReadOnlyDictionary <ushort, IChannel> handlers, PipeReader reader)
{
while (true)
{
ReadResult result = await reader.ReadAsync();
ReadOnlySequence <byte> buffer = result.Buffer;
while (buffer.TryReadFrame(out Frame frame))
{
//TODO: proper memory management instead of copy
var slice = buffer.Slice(Constants.HeaderSize, frame.ContentLength);
await handlers[frame.Channel].Handle(frame, ref slice);
var nextPart = buffer.GetPosition(frame.TotalLength);
buffer = buffer.Slice(nextPart);
}
;
reader.AdvanceTo(buffer.Start, buffer.End);
if (result.IsCompleted)
{
break;
}
}
reader.Complete();
}
public void GetOffset_MultiSegment_InvalidSequencePositionSegment()
{
ReadOnlySequence <T> buffer = GetFourSegmentsReadOnlySequence();
ReadOnlySequence <T> buffer2 = new ReadOnlySequence <T>(new T[50]);
Assert.Throws <InvalidCastException>(() => buffer.GetOffset(buffer2.GetPosition(25)));
}
public void AdvanceTo_PositionFromUnrelatedSequence()
{
MockPool <char> mockPool = new MockPool <char>();
var seqA = new Sequence <char>(mockPool);
var seqB = new Sequence <char>(mockPool);
var mem1 = seqA.GetMemory(3);
mem1.Span.Fill('a');
seqA.Advance(mem1.Length);
var mem2 = seqB.GetMemory(3);
mem2.Span.Fill('b');
seqB.Advance(mem2.Length);
ReadOnlySequence <char> rosA = seqA;
ReadOnlySequence <char> rosB = seqB;
var posB = rosB.GetPosition(2);
Assert.Throws <ArgumentException>(() => seqA.AdvanceTo(posB));
Assert.Equal(3, seqA.AsReadOnlySequence.Length);
Assert.Equal(3, seqB.AsReadOnlySequence.Length);
}
private void HandlePacket(ReadOnlySequence <byte> buffer, MCConnectionContext ctx, IPacketQueue packetQueue)
{
var reader = _packetReaderFactory.CreateReader(buffer);
var length = reader.ReadVarInt();
if (length > reader.Buffer.Length || length < 1 /* 1 = small ID but no fields*/)
{
_logger.LogCritical($"Read Invalid length {length:X}. Aborting");
ctx.Abort();
return;
}
var lengthLength = buffer.Length - reader.Buffer.Length;
reader = new MCPacketReader(reader.Buffer.Slice(0, length));
var id = reader.ReadVarInt();
using var packetIdScope = _logger.BeginScope($"Packet ID: {id:x2}");
_packetHandler.HandlePacket(ctx, reader, packetQueue, id);
// NOT IDEAL, but easiest
var packetSize = length + lengthLength;
ctx.Transport.Input.AdvanceTo(buffer.GetPosition(packetSize));
_metrics.Measure.Histogram.Update(MetricsRegistry.ReadPacketSize, packetSize);
}
private bool ParsePreface(ReadOnlySequence <byte> readableBuffer, out SequencePosition consumed, out SequencePosition examined)
{
consumed = readableBuffer.Start;
examined = readableBuffer.End;
if (readableBuffer.Length < ClientPreface.Length)
{
return(false);
}
var span = readableBuffer.IsSingleSegment
? readableBuffer.First.Span
: readableBuffer.ToSpan();
for (var i = 0; i < ClientPreface.Length; i++)
{
if (ClientPreface[i] != span[i])
{
throw new Http2ConnectionErrorException(CoreStrings.Http2ErrorInvalidPreface, Http2ErrorCode.PROTOCOL_ERROR);
}
}
consumed = examined = readableBuffer.GetPosition(ClientPreface.Length);
return(true);
}
static bool TryParseLines(
ref ReadOnlySequence <byte> buffer,
out string message)
{
SequencePosition?position;
StringBuilder outputMessage = new();
while (true)
{
position = buffer.PositionOf((byte)'\n');
if (!position.HasValue)
{
break;
}
outputMessage.Append(Encoding.ASCII.GetString(buffer.Slice(buffer.Start, position.Value)))
.AppendLine();
buffer = buffer.Slice(buffer.GetPosition(1, position.Value));
}
;
message = outputMessage.ToString();
return(message.Length != 0);
}
private static void Byte_MultiSegment_PosLong(int bufSize, int bufOffset)
{
var segment1 = new BufferSegment <byte>(new byte[bufSize / 10]);
BufferSegment <byte> segment2 = segment1;
for (int j = 0; j < 10; j++)
{
segment2 = segment2.Append(new byte[bufSize / 10]);
}
var buffer = new ReadOnlySequence <byte>(segment1, bufOffset, segment2, bufSize / 10 - bufOffset);
long sliceLen = buffer.Length;
SequencePosition start = buffer.GetPosition(0);
foreach (BenchmarkIteration iteration in Benchmark.Iterations)
{
int localInt = 0;
using (iteration.StartMeasurement())
{
for (int i = 0; i < Benchmark.InnerIterationCount; i++)
{
ReadOnlySequence <byte> temp = buffer.Slice(start, sliceLen);
localInt ^= temp.Start.GetInteger();
}
}
_volatileInt = localInt;
}
}
private async Task ReadPipeAsync(PipeReader reader, CancellationToken cancellationToken = default)
{
while (true)
{
if (cancellationToken.IsCancellationRequested)
{
reader.CancelPendingRead();
await reader.CompleteAsync();
break;
}
bool isAny = reader.TryRead(out ReadResult result);
if (!isAny)
{
continue;
}
ReadOnlySequence <byte> buffer = result.Buffer;
// In the event that no message is parsed successfully, mark consumed
// as nothing and examined as the entire buffer.
SequencePosition consumed = buffer.Start;
SequencePosition examined = buffer.End;
try
{
if (parser.TryParse(ref buffer, out Message message, out int bConsumed, GetName()))
{
// A single message was successfully parsed so mark the start as the
// parsed buffer as consumed. TryParseMessage trims the buffer to
// point to the data after the message was parsed.
consumed = buffer.GetPosition(bConsumed);
// Examined is marked the same as consumed here, so the next call
// to ReadSingleMessageAsync will process the next message if there's
// one.
examined = consumed;
Console.WriteLine();
if (!(message is null))
{
foreach (var observer in observers)
{
observer.OnNext(message);
}
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
foreach (var observer in observers)
{
observer.OnError(e);
}
}
finally
{
reader.AdvanceTo(consumed, examined);
}
}
}
public void InputUdpMessage(ref ReadOnlySequence <byte> buffer)
{
var udpBuffer = buffer.Slice(buffer.GetPosition(4));
var reader = new WrappedReader(udpBuffer, this.Order, this.readerFlushDelegate);
this.Pipeline.OnTransportRead(this, ref reader);
reader.Flush();
}
public void GetPositionPrefersNextSegment()
{
BufferSegment <byte> bufferSegment1 = new BufferSegment <byte>(new byte[50]);
BufferSegment <byte> bufferSegment2 = bufferSegment1.Append(new byte[0]);
ReadOnlySequence <byte> buffer = new ReadOnlySequence <byte>(bufferSegment1, 0, bufferSegment2, 0);
SequencePosition c1 = buffer.GetPosition(50);
Assert.Equal(0, c1.GetInteger());
Assert.Equal(bufferSegment2, c1.GetObject());
c1 = buffer.GetPosition(50, buffer.Start);
Assert.Equal(0, c1.GetInteger());
Assert.Equal(bufferSegment2, c1.GetObject());
}
public void SegmentStartIsConsideredInBoundsCheck()
{
// 0 50 100 0 50 100
// [ ##############] -> [############## ]
// ^c1 ^c2
var bufferSegment1 = new BufferSegment(new byte[49]);
BufferSegment bufferSegment2 = bufferSegment1.Append(new byte[50]);
var buffer = new ReadOnlySequence <byte>(bufferSegment1, 0, bufferSegment2, 50);
SequencePosition c1 = buffer.GetPosition(buffer.Start, 25); // segment 1 index 75
SequencePosition c2 = buffer.GetPosition(buffer.Start, 55); // segment 2 index 5
ReadOnlySequence <byte> sliced = buffer.Slice(c1, c2);
Assert.Equal(30, sliced.Length);
}
public static bool TrySliceTo(this ReadOnlySequence <byte> buffer, Func <ReadOnlySequence <byte>, bool> checkFunc, out ReadOnlySequence <byte> slice, out SequencePosition position)
{
for (int i = 0; i < buffer.Length; ++i)
{
var woring = buffer.Slice(i, (buffer.Length - i));
if (checkFunc(woring))
{
slice = buffer.Slice(0, i);
position = buffer.GetPosition(i);
return(true);
}
}
slice = buffer;
position = buffer.GetPosition(0);
return(false);
}
private FrameReader(ref ReadOnlySequence <byte> sequence, Encoding encoding = null)
{
this.sequence = sequence;
position = sequence.GetPosition(0);
Encoding = encoding ?? DefaultEncoding;
Encoder = Encoding.GetEncoder();
}
internal bool TryReadOperation(ref ReadOnlySequence <byte> buffer, [MaybeNullWhen(false)] out IMemoryOwner <byte> operationResponse)
{
if (buffer.Length < HeaderOffsets.HeaderLength)
{
// Not enough data to read the body length from the header
operationResponse = null;
return(false);
}
int responseSize = HeaderOffsets.HeaderLength;
var sizeSegment = buffer.Slice(HeaderOffsets.BodyLength, sizeof(int));
if (sizeSegment.IsSingleSegment)
{
responseSize += ByteConverter.ToInt32(sizeSegment.First.Span);
}
else
{
// Edge case, we're split across segments in the buffer
Span <byte> tempSpan = stackalloc byte[sizeof(int)];
sizeSegment.CopyTo(tempSpan);
responseSize += ByteConverter.ToInt32(tempSpan);
}
if (buffer.Length < responseSize)
{
// Insufficient data, keep filling the buffer
operationResponse = null;
return(false);
}
// Slice to get operationBuffer, which is just the operation
// And slice the original buffer to start after this operation,
// we pass this back by ref so it's ready for the next operation
var position = buffer.GetPosition(responseSize);
var operationBuffer = buffer.Slice(0, position);
buffer = buffer.Slice(position);
// Copy the response to a separate, contiguous memory buffer
operationResponse = MemoryPool <byte> .Shared.RentAndSlice(responseSize);
try
{
operationBuffer.CopyTo(operationResponse.Memory.Span);
return(true);
}
catch
{
// Cleanup the memory in case of exception
operationResponse.Dispose();
throw;
}
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="buffer">The buffer to read from.</param>
public TokenReader(ReadOnlySequence <byte> buffer)
{
_buffer = buffer;
_spanPosition = _buffer.GetPosition(0);
_span = default;
_spanIndex = 0;
_peek = default;
_hasPeeked = false;
}
public void GetOffset_SingleSegment_Slice()
{
ReadOnlySequence <T> buffer = new ReadOnlySequence <T>(new T[50]);
for (int i = 0; i <= buffer.Length; i++)
{
Assert.Equal(buffer.Slice(0, i).Length, buffer.GetOffset(buffer.GetPosition(i)));
}
}
private static (JsonReaderState, SequencePosition, string) ProcessDataSpan(ReadOnlySequence <byte> ros, bool isFinalBlock, JsonReaderState state = default)
{
var builder = new StringBuilder();
ReadOnlySpan <byte> leftOver = default;
byte[] pooledArray = null;
JsonUtf8Reader json = default;
long totalConsumed = 0;
foreach (ReadOnlyMemory <byte> mem in ros)
{
ReadOnlySpan <byte> span = mem.Span;
if (leftOver.Length > int.MaxValue - span.Length)
{
throw new ArgumentOutOfRangeException("Current sequence segment size is too large to fit left over data from the previous segment into a 2 GB buffer.");
}
pooledArray = ArrayPool <byte> .Shared.Rent(span.Length + leftOver.Length); // This is guaranteed to not overflow
Span <byte> bufferSpan = pooledArray.AsSpan(0, leftOver.Length + span.Length);
leftOver.CopyTo(bufferSpan);
span.CopyTo(bufferSpan.Slice(leftOver.Length));
json = new JsonUtf8Reader(bufferSpan, isFinalBlock, state);
while (json.Read())
{
switch (json.TokenType)
{
case JsonTokenType.PropertyName:
json.TryGetValueAsString(out string value);
builder.Append(value);
break;
}
}
if (json.BytesConsumed < bufferSpan.Length)
{
leftOver = bufferSpan.Slice((int)json.BytesConsumed);
}
else
{
leftOver = default;
}
totalConsumed += json.BytesConsumed;
Assert.Equal(json.BytesConsumed, json.CurrentState.BytesConsumed);
Assert.Equal(json.Position, json.CurrentState.Position);
if (pooledArray != null) // TODO: Will this work if data spans more than two segments?
{
ArrayPool <byte> .Shared.Return(pooledArray);
}
state = json.CurrentState;
}
return(json.CurrentState, ros.GetPosition(totalConsumed), builder.ToString());
}
private FrameReader Reset(ref ReadOnlySequence <byte> value)
{
sequence = value;
position = sequence.GetPosition(0);
Encoder.Reset();
return(this);
}
// Adapted from https://devblogs.microsoft.com/dotnet/system-io-pipelines-high-performance-io-in-net/
public static async Task ReadLinesUsingPipesAsync(this Stream stream, Func <ReadOnlySequence <byte>, Task> callback, CancellationToken token = default, char separator = '\n')
{
var pipe = new Pipe();
try
{
await StressTaskExtensions.WhenAllThrowOnFirstException(token, FillPipeAsync, ReadPipeAsync);
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
}
async Task FillPipeAsync(CancellationToken token)
{
try
{
await stream.CopyToAsync(pipe.Writer, token);
}
catch (Exception e)
{
pipe.Writer.Complete(e);
throw;
}
pipe.Writer.Complete();
}
async Task ReadPipeAsync(CancellationToken token)
{
while (!token.IsCancellationRequested)
{
ReadResult result = await pipe.Reader.ReadAsync(token);
ReadOnlySequence <byte> buffer = result.Buffer;
SequencePosition? position;
do
{
position = buffer.PositionOf((byte)separator);
if (position != null)
{
await callback(buffer.Slice(0, position.Value));
buffer = buffer.Slice(buffer.GetPosition(1, position.Value));
}
}while (position != null);
pipe.Reader.AdvanceTo(buffer.Start, buffer.End);
if (result.IsCompleted)
{
break;
}
}
}
}
public void GetPositionDoesNotCrossOutsideBuffer()
{
var bufferSegment1 = new BufferSegment <byte>(new byte[100]);
BufferSegment <byte> bufferSegment2 = bufferSegment1.Append(new byte[100]);
BufferSegment <byte> bufferSegment3 = bufferSegment2.Append(new byte[0]);
var buffer = new ReadOnlySequence <byte>(bufferSegment1, 0, bufferSegment2, 100);
SequencePosition c1 = buffer.GetPosition(200);
Assert.Equal(100, c1.GetInteger());
Assert.Equal(bufferSegment2, c1.GetObject());
c1 = buffer.GetPosition(200, buffer.Start);
Assert.Equal(100, c1.GetInteger());
Assert.Equal(bufferSegment2, c1.GetObject());
}
public void GetOffset_MultiSegment_Slice()
{
ReadOnlySequence <T> buffer = GetFourSegmentsReadOnlySequence();
for (int i = 0; i <= buffer.Length; i++)
{
Assert.Equal(buffer.Slice(0, i).Length, buffer.GetOffset(buffer.GetPosition(i)));
}
}
public void SeekSkipsEmptySegments()
{
var bufferSegment1 = new BufferSegment <T>(new T[100]);
BufferSegment <T> bufferSegment2 = bufferSegment1.Append(new T[0]);
BufferSegment <T> bufferSegment3 = bufferSegment2.Append(new T[0]);
BufferSegment <T> bufferSegment4 = bufferSegment3.Append(new T[100]);
var buffer = new ReadOnlySequence <T>(bufferSegment1, 0, bufferSegment4, 100);
SequencePosition c1 = buffer.GetPosition(100);
Assert.Equal(0, c1.GetInteger());
Assert.Equal(bufferSegment4, c1.GetObject());
c1 = buffer.GetPosition(100, buffer.Start);
Assert.Equal(0, c1.GetInteger());
Assert.Equal(bufferSegment4, c1.GetObject());
}