public bool TakeMessageHeaders(ref SequenceReader <byte> reader, bool trailers)
{
// Make sure the buffer is limited
if (reader.Remaining > _remainingRequestHeadersBytesAllowed)
{
// Input oversize, cap amount checked
return(TrimAndTakeMessageHeaders(ref reader, trailers));
}
var alreadyConsumed = reader.Consumed;
try
{
var result = _parser.ParseHeaders(new Http1ParsingHandler(this, trailers), ref reader);
if (result)
{
TimeoutControl.CancelTimeout();
}
return(result);
}
finally
{
_remainingRequestHeadersBytesAllowed -= reader.Consumed - alreadyConsumed;
}
bool TrimAndTakeMessageHeaders(ref SequenceReader <byte> reader, bool trailers)
{
var trimmedBuffer = reader.Sequence.Slice(reader.Position, _remainingRequestHeadersBytesAllowed);
var trimmedReader = new SequenceReader <byte>(trimmedBuffer);
try
{
if (!_parser.ParseHeaders(new Http1ParsingHandler(this, trailers), ref trimmedReader))
{
// We read the maximum allowed but didn't complete the headers.
KestrelBadHttpRequestException.Throw(RequestRejectionReason.HeadersExceedMaxTotalSize);
}
TimeoutControl.CancelTimeout();
reader.Advance(trimmedReader.Consumed);
return(true);
}
finally
{
_remainingRequestHeadersBytesAllowed -= trimmedReader.Consumed;
}
}
}
private Task <SequencePosition> Consume(ReadOnlySequence <byte> data,
Action <WitFrame> onFrame)
{
if (data.Length < _minimumReadBytes)
{
return(Task.FromResult(ZeroPosition));
}
var buffer = new byte[_minimumReadBytes];
buffer[0] = _seq1[0];
buffer[1] = _seq1[1];
var buf = buffer.AsSpan().Slice(2);
var reader = new SequenceReader <byte>(data);
while (!reader.End)
{
if (!_isSync)
{
if (!reader.TryReadTo(out ReadOnlySequence <byte> _, _seq1.AsSpan(), true))
{
// remove useless data
reader.Advance(reader.Length);
return(Task.FromResult(reader.Position));
}
_isSync = true;
return(Task.FromResult(reader.Position));
}
if (reader.TryReadTo(out ReadOnlySequence <byte> sequence, _seq1.AsSpan(), true))
{
if (sequence.Length != _minimumReadBytes - _seq1.Length)
{
// broken packet
return(Task.FromResult(reader.Position));
}
sequence.CopyTo(buf);
onFrame(new WitFrame(_clock.Elapsed, buffer));
return(Task.FromResult(reader.Position));
}
Debug.WriteLine($"Wait {sequence.Length}");
return(Task.FromResult(reader.Position));
}
// serial connection has been closed
Debug.WriteLine("End");
return(Task.FromResult(ZeroPosition));
}
public void Test1()
{
var reader = new ReadOnlySequence <byte>(new byte[] { 0x7E, 0, 1, 2, 0x7E });
SequenceReader <byte> seqReader = new SequenceReader <byte>(reader);
int index = 0;
byte mark = 0;
long totalConsumed = 0;
List <byte[]> packages = new List <byte[]>();
while (!seqReader.End)
{
if (seqReader.IsNext(0x7E, advancePast: true))
{
if (mark == 1)
{
var package = seqReader.Sequence.Slice(totalConsumed, seqReader.Consumed - totalConsumed).ToArray();
packages.Add(package);
totalConsumed += (seqReader.Consumed - totalConsumed);
index++;
if (seqReader.End)
{
break;
}
seqReader.Advance(1);
mark = 0;
}
mark++;
}
else
{
seqReader.Advance(1);
}
index++;
}
Assert.Equal(5, index);
Assert.Single(packages);
Assert.Equal(5, seqReader.Consumed);
}
private void ProcessSegments(ref ReadOnlySequence <byte> buffer)
{
var sequenceReader = new SequenceReader <byte>(buffer);
while (!sequenceReader.End)
{
while (sequenceReader.TryReadTo(out ReadOnlySequence <byte> line, m_NewLineBytes))
{
m_Writer.WriteAsync(Encoding.UTF8.GetString(line), m_Token).GetAwaiter();
}
buffer = buffer.Slice(sequenceReader.Position);
sequenceReader.Advance(buffer.Length);
}
}
private void LoadRequestBody(ref SequenceReader <byte> reader)
{
if (_httpRequestInfo.FilterStatus == FilterStatus.LoadingBody)
{
var position = reader.Consumed;
var rest = reader.Length - position;
if (rest >= _httpRequestInfo.ContentLength)
{
_httpRequestInfo.Body = reader.Sequence.Slice(position, _httpRequestInfo.ContentLength).ToArray();
reader.Advance(rest);//多余的不要了
_httpRequestInfo.FilterStatus = FilterStatus.Completed;
}
}
}
protected internal override void DecodeBody(ref SequenceReader <byte> reader, object context)
{
TableID = reader.ReadLong(6);
reader.Advance(2); // skip flags
byte len;
reader.TryRead(out len);
SchemaName = reader.ReadString(len);
reader.TryRead(out len);// 0x00
reader.TryRead(out len);
TableName = reader.ReadString(len);
reader.TryRead(out len);// 0x00
ColumnCount = (int)reader.ReadLengthEncodedInteger();
ColumnTypes = reader.Sequence.Slice(reader.Consumed, ColumnCount).ToArray();
reader.Advance(ColumnCount);
reader.ReadLengthEncodedInteger();
ColumnMetadata = ReadColumnMetadata(ref reader, ColumnTypes);
NullBitmap = reader.ReadBitArray(ColumnCount);
RebuildReaderAsCRC(ref reader);
Metadata = ReadTableMetadata(ref reader);
if (context is ReplicationState repState)
{
repState.TableMap[TableID] = this;
}
}
private static string ProcessLine(ref ReadOnlySequence <byte> buffer)
{
string str = null;
if (buffer.IsSingleSegment)
{
var span = buffer.FirstSpan;
int consumed;
while (span.Length > 0)
{
var newLine = span.IndexOf(NewLine);
if (newLine == -1)
{
break;
}
var line = span.Slice(0, newLine);
str += Encoding.UTF8.GetString(line) + Environment.NewLine;
// simulate string processing
//str = str.AsSpan().Slice(0, 10).ToString();
consumed = line.Length + NewLine.Length;
span = span.Slice(consumed);
buffer = buffer.Slice(consumed);
}
}
else
{
var sequenceReader = new SequenceReader <byte>(buffer);
while (!sequenceReader.End)
{
while (sequenceReader.TryReadTo(out ReadOnlySequence <byte> line, NewLine))
{
str += Encoding.UTF8.GetString(line) + Environment.NewLine;
// simulate string processing
//str = str.AsSpan().Slice(0, 10).ToString();
}
buffer = buffer.Slice(sequenceReader.Position);
sequenceReader.Advance(buffer.Length);
}
}
return(str);
}
public override TPackageInfo Filter(ref SequenceReader <byte> reader)
{
if (!_foundHeader)
{
if (reader.Length < _headerSize)
{
return(null);
}
var header = reader.Sequence.Slice(0, _headerSize);
var bodyLength = GetBodyLengthFromHeader(header);
if (bodyLength < 0)
{
throw new ProtocolException("Failed to get body length from the package header.");
}
// rewind to the original postion
// don't consume any data from the reader
reader.Rewind(reader.Position.GetInteger());
if (bodyLength == 0)
{
return(DecodePackage(header));
}
_foundHeader = true;
_totalSize = _headerSize + bodyLength;
return(Filter(ref reader));
}
var totalSize = _totalSize;
var sequence = reader.Sequence;
if (reader.Length > totalSize)
{
sequence = sequence.Slice(0, totalSize);
}
var package = DecodePackage(sequence);
// mark the data consumed
reader.Advance(totalSize);
return(package);
}
public ITokenPatternMatch?GetFirstMatch(ref ReadOnlySequence <byte> buffer, long startIndex, char?escape)
{
TokenPatternMatch?result = null;
SequenceReader <byte> sequenceReader = new SequenceReader <byte>(buffer);
sequenceReader.Advance(startIndex);
ReadOnlySpan <byte> givenPattern = new ReadOnlySpan <byte>(this.Value);
bool hasRead = true;
(bool Result, long Index)match = (false, -1);
if (escape is null)
{
do
{
hasRead = sequenceReader.TryReadTo(out _, givenPattern, false);
if (hasRead == true)
{
match = this.GetMatch(ref sequenceReader);
}
}while(match.Result == false && hasRead == true);
}
else
{
do
{
hasRead = sequenceReader.TryReadTo(out _, givenPattern, false);
if (hasRead == true)
{
if (this.IsEscaped(escape.Value, ref sequenceReader) == false)
{
match = this.GetMatch(ref sequenceReader);
}
}
}while(match.Result == false && hasRead == true);
}
if (match.Result == true)
{
result = new TokenPatternMatch(this, match.Index);
}
return(result);
}
public override bool Process(QueryResult package, ref SequenceReader <byte> reader, out IPackagePartReader nextPartReader, out bool needMoreData)
{
nextPartReader = null;
if (!reader.TryReadTo(out ReadOnlySequence <byte> data, 0x00, false))
{
needMoreData = true;
return(false);
}
reader.Advance(1);
package.ErrorMessage = data.GetString(Encoding.UTF8);
needMoreData = false;
return(true);
}
public ProxyDataPackageInfo Filter(ref SequenceReader <byte> reader)
{
var terminatorSpan = _headerTerminator.Span;
if (!reader.TryReadTo(out ReadOnlySequence <byte> pack, terminatorSpan, advancePastDelimiter: false))
{
return(null);
}
reader.Advance(terminatorSpan.Length);
return(new ProxyDataPackageInfo {
Data = pack.ToArray()
});
}
public (ReaderState?, bool?) ReadChart(ref SequenceReader <byte> reader, CancellationToken cancellationToken, BsbChartReaderState state)
{
if (reader.IsNext(TextSegmentEndToken.Span))
{
reader.Advance(TextSegmentEndToken.Length);
this.FlushEntry(state);
return(ReaderState.BitDepth, null);
}
if (reader.TryReadTo(out ReadOnlySequence <byte> text, TextEntryEndToken.Span))
{
// TODO: Use SequenceReader to extract segments?
string line = Encoding.ASCII.GetString(text);
if (line.Length > 0)
{
if (line[0] == '!')
{
this.StartEntry("!", line.Substring(1), state);
}
else if (line.StartsWith(" "))
{
this.lines.Add(line.Substring(4));
}
else
{
int index = line.IndexOf('/');
if (index >= 0)
{
this.StartEntry(line.Substring(0, index), line.Substring(index + 1), state);
}
else
{
this.StartEntry("?", line, state);
}
}
}
return(null, true);
}
else
{
return(null, false);
}
}
public override bool Process(WebSocketPackage package, ref SequenceReader <byte> reader, out IDataFramePartReader nextPartReader)
{
int required = 4;
if (reader.Length < required)
{
nextPartReader = this;
return(false);
}
package.MaskKey = reader.Sequence.Slice(0, 4).ToArray();
reader.Advance(4);
nextPartReader = PayloadDataReader;
return(false);
}
private Dictionary <int, int> ReadIntegerDictionary(ref SequenceReader <byte> reader)
{
var dict = new Dictionary <int, int>();
while (reader.TryPeek(out byte top) && top > 0)
{
dict[(int)reader.ReadLengthEncodedInteger()] = (int)reader.ReadLengthEncodedInteger();
}
if (reader.Remaining > 0)
{
reader.Advance(1);
}
return(dict);
}
private List <int> ReadIntegers(ref SequenceReader <byte> reader)
{
var charsets = new List <int>();
while (reader.TryPeek(out byte top) && top > 0)
{
charsets.Add((int)reader.ReadLengthEncodedInteger());
}
if (reader.Remaining > 0)
{
reader.Advance(1);
}
return(charsets);
}
static uint ReadUInt32BigEndian(ref SequenceReader <byte> reader)
{
using (var memory = MemoryPool <byte> .Shared.Rent(4))
{
var buffer = memory.Memory.Span.Slice(0, 4);
if (!reader.TryCopyTo(buffer))
{
throw new TransactionSerializationException("Incomplete data.");
}
reader.Advance(buffer.Length);
return(BinaryPrimitives.ReadUInt32BigEndian(buffer));
}
}
internal byte[] ReadData(ref SequenceReader <byte> reader, long amount, bool advance = true)
{
var data = new byte[amount];
if (!reader.TryCopyTo(data))
{
throw new InvalidOperationException("Failed to copy specified amount of data");
}
if (advance)
{
reader.Advance(amount);
}
return(data);
}
public void RewindEmptyFirstSpan()
{
// This is to hit the "if (memory.Length == 0)" branch in ResetReader.
ReadOnlySequence <byte> bytes = SequenceFactory.Create(new byte[][] {
new byte[0],
new byte[] { 1, 2 },
new byte[] { 3, 4 }
});
var reader = new SequenceReader <byte>(bytes);
reader.Advance(3);
Assert.True(reader.IsNext(4));
reader.Rewind(2);
Assert.Equal(new byte[] { 1, 2 }, reader.CurrentSpan.ToArray());
}
private DataSet ReadTrainingSet(ReadOnlySequence <byte> row)
{
var inputs = new double[784];
var outputs = new double[10];
var separator = (byte)',';
var sequenceReader = new SequenceReader <byte>(row);
if (!sequenceReader.TryReadTo(out ReadOnlySpan <byte> digitBytes, separator) ||
!Utf8Parser.TryParse(digitBytes, out int digit, out var _))
{
throw new InvalidOperationException("Cannot parse label.");
}
outputs[digit] = 1.0;
for (var inputIndex = 0; !sequenceReader.End; inputIndex++)
{
if (sequenceReader.TryReadTo(out ReadOnlySpan <byte> bytes, separator))
{
if (!Utf8Parser.TryParse(bytes, out int pixelValue, out var _))
{
throw new InvalidOperationException("Cannot parse pixel information.");
}
inputs[inputIndex] = AdjustValue(pixelValue);
}
else if (!sequenceReader.End)
{
var unreadSpan = sequenceReader.UnreadSpan;
if (!Utf8Parser.TryParse(unreadSpan, out int pixelValue, out var _))
{
throw new InvalidOperationException("Cannot parse pixel information.");
}
sequenceReader.Advance(unreadSpan.Length);
inputs[inputIndex] = AdjustValue(pixelValue);
}
else
{
break;
}
}
return(new DataSet(inputs, outputs));
}
public WebSocketPackage Filter(ref SequenceReader <byte> reader)
{
var terminatorSpan = _headerTerminator.Span;
if (!reader.TryReadTo(out ReadOnlySequence <byte> pack, terminatorSpan, advancePastDelimiter: false))
{
return(null);
}
reader.Advance(terminatorSpan.Length);
var package = ParseHandshake(ref pack);
NextFilter = new WebSocketDataPipelineFilter(package.HttpHeader);
return(package);
}
public override TPackageInfo Filter(ref SequenceReader <byte> reader)
{
if (reader.Length < _size)
{
return(null);
}
var pack = reader.Sequence.Slice(0, _size);
if (!CanDecodePackage(pack))
{
return(null);
}
reader.Advance(_size);
return(DecodePackage(pack));
}
private StringBuilder GetResponse(ReadResult result)
{
SequenceReader <byte> reader = new SequenceReader <byte>(result.Buffer);
reader.AdvancePast(0);
StringBuilder sb = new StringBuilder();
while (reader.Remaining > 0)
{
ReadOnlySpan <byte> unread = reader.UnreadSpan;
sb.Append(Encoding.UTF8.GetString(unread).Trim('\0'));
reader.Advance(unread.Length);
}
return(sb);
}
public static bool TryRead(ReadOnlySequence <byte> sequence, out uint version, IBufferWriter <byte> writer)
{
var reader = new SequenceReader <byte>(sequence);
if (!Varint.TryGetUInt32(ref reader, out version))
{
return(false);
}
if (!Varint.TryGetUInt32(ref reader, out var convertCompressionAlgorithmValue))
{
return(false);
}
var convertCompressionAlgorithm = (ConvertCompressionAlgorithm)convertCompressionAlgorithmValue;
if (convertCompressionAlgorithm == ConvertCompressionAlgorithm.Brotli)
{
using var decoder = default(BrotliDecoder);
for (; ;)
{
var status = decoder.Decompress(reader.UnreadSpan, writer.GetSpan(), out var bytesConsumed, out var bytesWritten);
if (status == OperationStatus.InvalidData)
{
_logger.Warn("invalid data");
return(false);
}
reader.Advance(bytesConsumed);
writer.Advance(bytesWritten);
if (status == OperationStatus.Done || (bytesConsumed == 0 && bytesWritten == 0))
{
break;
}
}
return(true);
}
else
{
_logger.Warn("not supported format");
return(false);
}
}
public override bool Process(WebSocketPackage package, ref SequenceReader <byte> reader, out IDataFramePartReader nextPartReader, out bool needMoreData)
{
nextPartReader = null;
long required = package.PayloadLength;
if (reader.Length < required)
{
needMoreData = true;
return(false);
}
needMoreData = false;
package.Data = reader.Sequence.Slice(0, required);
reader.Advance(required);
return(true);
}
internal protected override void DecodeBody(ref SequenceReader <byte> reader, object context)
{
Qos = (this.Flags - 0x30) / 2;
reader.TryReadBigEndian(out short protocolNameLen);
TopicName = reader.ReadString(protocolNameLen, Encoding.UTF8);
if (Qos > 0)
{
reader.TryRead(out byte msb);
reader.TryRead(out byte lsb1);
reader.TryPeek(out byte lsb2);
if (lsb2 == 0)
{
reader.Advance(1);
}
}
TopicBody = reader.ReadString((int)reader.Remaining, Encoding.UTF8);
TopicData = reader.Sequence.ToArray();
}
public static ReadOnlySequence <byte> TrimEnd(this ReadOnlySequence <byte> buffer)
{
var reader = new SequenceReader <byte>(buffer);
// Start at the end of the string
reader.Advance(reader.Length);
while (reader.CurrentSpan[reader.CurrentSpanIndex - 1] == (byte)' ' || reader.CurrentSpan[reader.CurrentSpanIndex - 1] == (byte)'\t' || reader.CurrentSpan[reader.CurrentSpanIndex - 1] == (byte)'\r')
{
if (reader.Position.GetInteger() <= 0)
{
break;
}
reader.Rewind(1);
}
return(reader.Sequence.Slice(0, reader.Position));
}
public override Elysium.Transaction Deserialize(
BitcoinAddress?sender,
BitcoinAddress?receiver,
ref SequenceReader <byte> reader,
int version)
{
var length = Convert.ToInt32(reader.Remaining);
using (var memory = MemoryPool <byte> .Shared.Rent(length))
{
var buffer = memory.Memory.Span.Slice(0, length);
reader.TryCopyTo(buffer);
reader.Advance(length);
return(this.StubbedDeserialize.Object(sender, receiver, buffer.ToArray(), version));
}
}
public SequencePointBlock(NettraceBlock block)
{
if (!Equals(block.Type.Name, KnownTypeNames.SPBlock))
{
throw new ArgumentException("Requires a SPBlock");
}
var reader = new SequenceReader <byte>(block.BlockBody);
_ = reader.TryReadLittleEndian(out Timestamp);
_ = reader.TryReadLittleEndian(out ThreadCount);
Threads = new SequencePointThread[ThreadCount];
for (int i = 0; i < Threads.Length; i++)
{
var span = MemoryMarshal.Cast <SequencePointThread, byte>(MemoryMarshal.CreateSpan(ref Threads[i], 1));
_ = reader.TryCopyTo(span);
reader.Advance(span.Length);
}
}
public RemoteProcedureCallMetadata(ReadOnlySequence <byte> metadata)
{
var reader = new SequenceReader <byte>(metadata);
if (!reader.TryReadBigEndian(out UInt16 version))
{
throw new ArgumentOutOfRangeException(nameof(version), $"Invalid RPC Metadata.");
}
if (version != VERSION)
{
throw new ArgumentOutOfRangeException(nameof(version), $"Unsupported RPC Version {version}, expected {VERSION}.");
}
if (!reader.TryReadBigEndian(out UInt16 servicelength))
{
throw new ArgumentOutOfRangeException(nameof(servicelength), $"Invalid RPC Metadata.");
}
if (reader.TryRead(out string service, servicelength))
{
Service = service;
}
else
{
throw new ArgumentOutOfRangeException(nameof(service), $"Invalid RPC Metadata.");
}
if (!reader.TryReadBigEndian(out UInt16 methodlength))
{
throw new ArgumentOutOfRangeException(nameof(methodlength), $"Invalid RPC Metadata.");
}
if (reader.TryRead(out string method, methodlength))
{
Method = method;
}
else
{
throw new ArgumentOutOfRangeException(nameof(method), $"Invalid RPC Metadata.");
}
if (!reader.TryReadBigEndian(out UInt16 tracinglength))
{
throw new ArgumentOutOfRangeException(nameof(tracinglength), $"Invalid RPC Metadata.");
}
Tracing = reader.Sequence.Slice(reader.Position, tracinglength);
reader.Advance(tracinglength);
Metadata = reader.Sequence.Slice(reader.Position, reader.Remaining);
}
public override TPackageInfo Filter(ref SequenceReader <byte> reader)
{
if (!_foundHeader)
{
if (reader.Length < _headerSize)
{
return(null);
}
var header = reader.Sequence.Slice(0, _headerSize);
var bodyLength = GetBodyLengthFromHeader(header);
if (bodyLength < 0)
{
throw new ProtocolException("Failed to get body length from the package header.");
}
if (bodyLength == 0)
{
return(DecodePackage(header));
}
_foundHeader = true;
_totalSize = _headerSize + bodyLength;
}
var totalSize = _totalSize;
if (reader.Length < totalSize)
{
return(null);
}
var pack = reader.Sequence.Slice(0, totalSize);
try
{
return(DecodePackage(pack));
}
finally
{
reader.Advance(totalSize);
}
}