/// <summary>
/// Read data from pipeline when available, constructing new RCON packets
/// </summary>
/// <param name="reader"></param>
/// <returns>Consumer Task</returns>
async Task ReadPipeAsync(PipeReader reader)
{
byte[] byteArr = new byte[Constants.MAX_PACKET_SIZE];
while (true)
{
ReadResult result = await reader.ReadAsync();
ReadOnlySequence <byte> buffer = result.Buffer;
SequencePosition packetStart = buffer.Start;
if (buffer.Length < 4)
{
if (result.IsCompleted)
{
break;
}
reader.AdvanceTo(packetStart, buffer.End);
continue;
// Complete header not yet received
}
int size = BitConverter.ToInt32(buffer.Slice(packetStart, 4).ToArray(), 0);
if (buffer.Length >= size + 4)
{
// Get packet end positions
SequencePosition packetEnd = buffer.GetPosition(size + 4, packetStart);
byteArr = buffer.Slice(packetStart, packetEnd).ToArray();
RCONPacket packet = RCONPacket.FromBytes(byteArr);
if (packet.Type == PacketType.AuthResponse)
{
// Failed auth responses return with an ID of -1
if (packet.Id == -1)
{
_authenticationTask.SetException(
new AuthenticationException($"Authentication failed for {_tcp.RemoteEndPoint}.")
);
}
// Tell Connect that authentication succeeded
_authenticationTask.SetResult(true);
}
// Forward rcon packet to handler
RCONPacketReceived(packet);
reader.AdvanceTo(packetEnd);
}
else
{
reader.AdvanceTo(packetStart, buffer.End);
}
// Tell the PipeReader how much of the buffer we have consumed
// Stop reading if there's no more data coming
if (buffer.IsEmpty && result.IsCompleted)
{
break; // exit loop
}
}
// If authentication did not complete
_authenticationTask.TrySetException(
new AuthenticationException($"Server did not respond to auth {_tcp.RemoteEndPoint}.")
);
// Mark the PipeReader as complete
await reader.CompleteAsync();
}
public static ParsedMessage?TryParseMessage(ReadOnlySequence <byte> buf)
{
var be = Binary.BigEndian;
if (buf.Length < 4)
{
return(null);
}
var off = 4;
var len = be.GetUInt32(buf.Slice(0, 4).ToArray());
if (len == 0)
{
return new ParsedMessage {
Message = new KeepAlive(),
Position = buf.GetPosition(off)
}
}
;
if (len + off > buf.Length)
{
return(null);
}
var type = buf.Slice(off++, 1).FirstSpan[0];
IProtocolMessage?parsed = null;
switch (type)
{
case 0:
parsed = new Choke();
break;
case 1:
parsed = new Unchoke();
break;
case 2:
parsed = new Interested();
break;
case 3:
parsed = new NotInterested();
break;
case 4: {
var idx = be.GetUInt32(buf.Slice(off, 4).ToArray());
off += 4;
parsed = new Have {
PieceIndex = idx
};
break;
}
case 5: {
var bitfieldLen = (int)(len - 1);
var bits = buf.Slice(off, bitfieldLen).ToArray();
off += bitfieldLen;
parsed = new Bitfield {
Bits = bits
};
break;
}
case 6:
throw new NotImplementedException("Request");
case 7:
throw new NotImplementedException("Piece");
case 8:
throw new NotImplementedException("Cancel");
case 20:
throw new NotImplementedException("Extension Message");
default:
throw new ArgumentOutOfRangeException(nameof(type), $"Invalid Peer Message type {type}");
}
if (parsed != null)
{
return new ParsedMessage {
Message = parsed,
Position = buf.GetPosition(off)
}
}
;
return(null);
}
}
}
public ParseResult ParseMessage(ReadOnlySequence <byte> buffer, out SequencePosition consumed, out SequencePosition examined, out byte[] message)
{
consumed = buffer.Start;
examined = buffer.End;
message = null;
var start = consumed;
while (buffer.Length > 0)
{
if (!(buffer.PositionOf(ByteLF) is SequencePosition lineEnd))
{
// For the case of data: Foo\r\n\r\<Anything except \n>
if (_internalParserState == InternalParseState.ReadEndOfMessage)
{
if (ConvertBufferToSpan(buffer.Slice(start, buffer.End)).Length > 1)
{
throw new FormatException("Expected a \\r\\n frame ending");
}
}
// Partial message. We need to read more.
return(ParseResult.Incomplete);
}
lineEnd = buffer.GetPosition(1, lineEnd);
var line = ConvertBufferToSpan(buffer.Slice(start, lineEnd));
buffer = buffer.Slice(line.Length);
if (line.Length <= 1)
{
throw new FormatException("There was an error in the frame format");
}
// Skip comments
if (line[0] == ByteColon)
{
start = lineEnd;
consumed = lineEnd;
continue;
}
if (IsMessageEnd(line))
{
_internalParserState = InternalParseState.ReadEndOfMessage;
}
// To ensure that the \n was preceded by a \r
// since messages can't contain \n.
// data: foo\n\bar should be encoded as
// data: foo\r\n
// data: bar\r\n
else if (line[line.Length - SseLineEnding.Length] != ByteCR)
{
throw new FormatException("Unexpected '\\n' in message. A '\\n' character can only be used as part of the newline sequence '\\r\\n'");
}
else
{
EnsureStartsWithDataPrefix(line);
}
var payload = Array.Empty <byte>();
switch (_internalParserState)
{
case InternalParseState.ReadMessagePayload:
EnsureStartsWithDataPrefix(line);
// Slice away the 'data: '
var payloadLength = line.Length - (DataPrefix.Length + SseLineEnding.Length);
var newData = line.Slice(DataPrefix.Length, payloadLength).ToArray();
_data.Add(newData);
start = lineEnd;
consumed = lineEnd;
break;
case InternalParseState.ReadEndOfMessage:
if (_data.Count == 1)
{
payload = _data[0];
}
else if (_data.Count > 1)
{
// Find the final size of the payload
var payloadSize = 0;
foreach (var dataLine in _data)
{
payloadSize += dataLine.Length;
}
payloadSize += _newLine.Length * _data.Count;
// Allocate space in the payload buffer for the data and the new lines.
// Subtract newLine length because we don't want a trailing newline.
payload = new byte[payloadSize - _newLine.Length];
var offset = 0;
foreach (var dataLine in _data)
{
dataLine.CopyTo(payload, offset);
offset += dataLine.Length;
if (offset < payload.Length)
{
_newLine.CopyTo(payload, offset);
offset += _newLine.Length;
}
}
}
message = payload;
consumed = lineEnd;
examined = consumed;
return(ParseResult.Completed);
}
if (buffer.Length > 0 && buffer.First.Span[0] == ByteCR)
{
_internalParserState = InternalParseState.ReadEndOfMessage;
}
}
return(ParseResult.Incomplete);
}
/// <summary>
/// Searches for a byte in the <see cref="ReadOnlyBuffer"/> and returns a sliced <see cref="ReadOnlyBuffer"/> that
/// contains all data up to and excluding the byte, and a <see cref="SequencePosition"/> that points to the byte.
/// </summary>
/// <param name="b1">The first byte to search for</param>
/// <param name="slice">A <see cref="ReadOnlyBuffer"/> slice that contains all data up to and excluding the first byte.</param>
/// <param name="cursor">A <see cref="SequencePosition"/> that points to the second byte</param>
/// <returns>True if the byte sequence was found, false if not found</returns>
public static bool TrySliceTo(this ReadOnlySequence <byte> buffer, byte b1, out ReadOnlySequence <byte> slice, out SequencePosition cursor)
{
if (buffer.IsEmpty)
{
slice = default;
cursor = default;
return(false);
}
var byte0Vector = GetVector(b1);
var seek = 0;
foreach (var memory in buffer)
{
var currentSpan = memory.Span;
var found = false;
if (Vector.IsHardwareAccelerated)
{
while (currentSpan.Length >= VectorWidth)
{
var data = MemoryMarshal.Read <Vector <byte> >(currentSpan);
var byte0Equals = Vector.Equals(data, byte0Vector);
if (byte0Equals.Equals(Vector <byte> .Zero))
{
currentSpan = currentSpan.Slice(VectorWidth);
seek += VectorWidth;
}
else
{
var index = FindFirstEqualByte(ref byte0Equals);
seek += index;
found = true;
break;
}
}
}
if (!found)
{
// Slow search
for (int i = 0; i < currentSpan.Length; i++)
{
if (currentSpan[i] == b1)
{
found = true;
break;
}
seek++;
}
}
if (found)
{
cursor = buffer.GetPosition(seek);
slice = buffer.Slice(buffer.Start, cursor);
return(true);
}
}
slice = default;
cursor = default;
return(false);
}
public async Task OnConnectedAsync(FramingConnection connection)
{
TimeSpan receiveTimeout = connection.ServiceDispatcher.Binding.ReceiveTimeout;
var timeoutHelper = new TimeoutHelper(receiveTimeout);
bool success = false;
try
{
var decoder = connection.FramingDecoder as ServerSingletonDecoder;
Fx.Assert(decoder != null, "FramingDecoder must be non-null and an instance of ServerSessionDecoder");
// first validate our content type
//ValidateContentType(connection, decoder);
UpgradeState upgradeState = UpgradeState.None;
// next read any potential upgrades and finish consuming the preamble
ReadOnlySequence <byte> buffer = ReadOnlySequence <byte> .Empty;
while (true)
{
if (buffer.Length == 0 && CanReadAndDecode(upgradeState))
{
System.IO.Pipelines.ReadResult readResult = await connection.Input.ReadAsync();
buffer = readResult.Buffer;
if (readResult.IsCompleted)
{
throw DiagnosticUtility.ExceptionUtility.ThrowHelperError(decoder.CreatePrematureEOFException());
}
}
while (true)
{
if (CanReadAndDecode(upgradeState))
{
Fx.Assert(buffer.Length > 0, "There must be something in the buffer to decode");
int bytesDecoded = decoder.Decode(buffer);
if (bytesDecoded > 0)
{
buffer = buffer.Slice(bytesDecoded);
if (buffer.Length == 0)
{
connection.Input.AdvanceTo(buffer.Start);
}
}
}
switch (decoder.CurrentState)
{
case ServerSingletonDecoder.State.UpgradeRequest:
switch (upgradeState)
{
case UpgradeState.None:
//change the state so that we don't read/decode until it is safe
ChangeUpgradeState(ref upgradeState, UpgradeState.VerifyingUpgradeRequest);
break;
case UpgradeState.VerifyingUpgradeRequest:
if (connection.StreamUpgradeAcceptor == null)
{
await connection.SendFaultAsync(FramingEncodingString.UpgradeInvalidFault, timeoutHelper.RemainingTime(), TransportDefaults.MaxDrainSize);
throw DiagnosticUtility.ExceptionUtility.ThrowHelperError(
new ProtocolException(SR.Format(SR.UpgradeRequestToNonupgradableService, decoder.Upgrade)));
}
if (!connection.StreamUpgradeAcceptor.CanUpgrade(decoder.Upgrade))
{
await connection.SendFaultAsync(FramingEncodingString.UpgradeInvalidFault, timeoutHelper.RemainingTime(), TransportDefaults.MaxDrainSize);
throw DiagnosticUtility.ExceptionUtility.ThrowHelperError(new ProtocolException(SR.Format(SR.UpgradeProtocolNotSupported, decoder.Upgrade)));
}
ChangeUpgradeState(ref upgradeState, UpgradeState.WritingUpgradeAck);
// accept upgrade
await connection.Output.WriteAsync(ServerSingletonEncoder.UpgradeResponseBytes, timeoutHelper.GetCancellationToken());
await connection.Output.FlushAsync(timeoutHelper.GetCancellationToken());
ChangeUpgradeState(ref upgradeState, UpgradeState.UpgradeAckSent);
break;
case UpgradeState.UpgradeAckSent:
// This state was used to capture any extra read bytes into PreReadConnection but we don't need to do that when using pipes.
// This extra state transition has been left here to maintain the same state transitions as on .NET Framework to make comparison easier.
ChangeUpgradeState(ref upgradeState, UpgradeState.BeginUpgrade);
break;
case UpgradeState.BeginUpgrade:
// Set input pipe so that the next read will return all the unconsumed bytes.
// If all bytes have already been consumed so the buffer has 0 length, AdvanceTo would throw
// as it's already been called.
if (buffer.Length > 0)
{
connection.Input.AdvanceTo(buffer.Start);
}
buffer = ReadOnlySequence <byte> .Empty;
try
{
await UpgradeConnectionAsync(connection);
ChangeUpgradeState(ref upgradeState, UpgradeState.EndUpgrade);
}
catch (Exception exception)
{
if (Fx.IsFatal(exception))
{
throw;
}
throw;
}
break;
case UpgradeState.EndUpgrade:
//Must be a different state here than UpgradeComplete so that we don't try to read from the connection
ChangeUpgradeState(ref upgradeState, UpgradeState.UpgradeComplete);
break;
case UpgradeState.UpgradeComplete:
//Client is doing more than one upgrade, reset the state
ChangeUpgradeState(ref upgradeState, UpgradeState.VerifyingUpgradeRequest);
break;
}
break;
case ServerSingletonDecoder.State.Start:
SetupSecurityIfNecessary(connection);
if (upgradeState == UpgradeState.UpgradeComplete || //We have done at least one upgrade, but we are now done.
upgradeState == UpgradeState.None) //no upgrade, just send the preample end bytes
{
ChangeUpgradeState(ref upgradeState, UpgradeState.WritingPreambleEnd);
// we've finished the preamble. Ack and return.
await connection.Output.WriteAsync(ServerSessionEncoder.AckResponseBytes);
await connection.Output.FlushAsync();
//terminal state
ChangeUpgradeState(ref upgradeState, UpgradeState.PreambleEndSent);
}
// If all bytes have already been consumed so the buffer has 0 length, AdvanceTo would throw
// as it's already been called.
if (buffer.Length > 0)
{
connection.Input.AdvanceTo(buffer.Start);
}
success = true;
await _next(connection);
return;
}
if (buffer.Length == 0)
{
break;
}
}
}
}
finally
{
if (!success)
{
connection.Abort();
}
}
}
public static void TestPartialJsonReaderMultiSegment(bool compactData, TestCaseType type, string jsonString)
{
// Skipping really large JSON since slicing them (O(n^2)) is too slow.
if (type == TestCaseType.Json40KB || type == TestCaseType.Json400KB || type == TestCaseType.ProjectLockJson ||
type == TestCaseType.DeepTree || type == TestCaseType.BroadTree || type == TestCaseType.LotsOfNumbers ||
type == TestCaseType.LotsOfStrings || type == TestCaseType.Json4KB)
{
return;
}
// Remove all formatting/indendation
if (compactData)
{
using (JsonTextReader jsonReader = new JsonTextReader(new StringReader(jsonString)))
{
jsonReader.FloatParseHandling = FloatParseHandling.Decimal;
JToken jtoken = JToken.ReadFrom(jsonReader);
var stringWriter = new StringWriter();
using (JsonTextWriter jsonWriter = new JsonTextWriter(stringWriter))
{
jtoken.WriteTo(jsonWriter);
jsonString = stringWriter.ToString();
}
}
}
byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString);
ReadOnlyMemory <byte> dataMemory = dataUtf8;
var sequences = new List <ReadOnlySequence <byte> >
{
new ReadOnlySequence <byte>(dataMemory)
};
for (int i = 0; i < dataUtf8.Length; i++)
{
var firstSegment = new BufferSegment <byte>(dataMemory.Slice(0, i));
ReadOnlyMemory <byte> secondMem = dataMemory.Slice(i);
BufferSegment <byte> secondSegment = firstSegment.Append(secondMem);
var sequence = new ReadOnlySequence <byte>(firstSegment, 0, secondSegment, secondMem.Length);
sequences.Add(sequence);
}
for (int i = 0; i < sequences.Count; i++)
{
var json = new Utf8JsonReader(sequences[i], isFinalBlock: true, default);
while (json.Read())
{
;
}
Assert.Equal(sequences[i].Length, json.BytesConsumed);
Assert.Equal(sequences[i].Length, json.CurrentState.BytesConsumed);
Assert.True(sequences[i].Slice(json.Position).IsEmpty);
Assert.True(sequences[i].Slice(json.CurrentState.Position).IsEmpty);
}
for (int i = 0; i < sequences.Count; i++)
{
ReadOnlySequence <byte> sequence = sequences[i];
for (int j = 0; j < dataUtf8.Length; j++)
{
var json = new Utf8JsonReader(sequence.Slice(0, j), isFinalBlock: false, default);
while (json.Read())
{
;
}
long consumed = json.BytesConsumed;
JsonReaderState jsonState = json.CurrentState;
byte[] consumedArray = sequence.Slice(0, consumed).ToArray();
Assert.Equal(consumedArray, sequence.Slice(0, json.Position).ToArray());
Assert.Equal(consumedArray, sequence.Slice(0, jsonState.Position).ToArray());
json = new Utf8JsonReader(sequence.Slice(consumed), isFinalBlock: true, json.CurrentState);
while (json.Read())
{
;
}
Assert.Equal(dataUtf8.Length - consumed, json.BytesConsumed);
Assert.Equal(json.BytesConsumed, json.CurrentState.BytesConsumed);
}
}
}
public Task OnDataAsync(Http2Frame dataFrame, ReadOnlySequence <byte> payload)
{
// Since padding isn't buffered, immediately count padding bytes as read for flow control purposes.
if (dataFrame.DataHasPadding)
{
// Add 1 byte for the padding length prefix.
OnDataRead(dataFrame.DataPadLength + 1);
}
var dataPayload = payload.Slice(0, dataFrame.DataPayloadLength); // minus padding
var endStream = dataFrame.DataEndStream;
if (dataPayload.Length > 0)
{
RequestBodyStarted = true;
if (endStream)
{
// No need to send any more window updates for this stream now that we've received all the data.
// Call before flushing the request body pipe, because that might induce a window update.
_inputFlowControl.StopWindowUpdates();
}
_inputFlowControl.Advance((int)dataPayload.Length);
lock (_completionLock)
{
if (IsAborted)
{
// Ignore data frames for aborted streams, but only after counting them for purposes of connection level flow control.
return(Task.CompletedTask);
}
// This check happens after flow control so that when we throw and abort, the byte count is returned to the connection
// level accounting.
if (InputRemaining.HasValue)
{
// https://tools.ietf.org/html/rfc7540#section-8.1.2.6
if (dataPayload.Length > InputRemaining.Value)
{
throw new Http2StreamErrorException(StreamId, CoreStrings.Http2StreamErrorMoreDataThanLength, Http2ErrorCode.PROTOCOL_ERROR);
}
InputRemaining -= dataPayload.Length;
}
foreach (var segment in dataPayload)
{
RequestBodyPipe.Writer.Write(segment.Span);
}
var flushTask = RequestBodyPipe.Writer.FlushAsync();
// It shouldn't be possible for the RequestBodyPipe to fill up an return an incomplete task if
// _inputFlowControl.Advance() didn't throw.
Debug.Assert(flushTask.IsCompleted);
}
}
if (endStream)
{
OnEndStreamReceived();
}
return(Task.CompletedTask);
}
internal static bool TryParseValue(
ref Utf8JsonReader reader,
[NotNullWhen(true)] out JsonDocument?document,
bool shouldThrow,
bool useArrayPools)
{
JsonReaderState state = reader.CurrentState;
CheckSupportedOptions(state.Options, nameof(reader));
// Value copy to overwrite the ref on an exception and undo the destructive reads.
Utf8JsonReader restore = reader;
ReadOnlySpan <byte> valueSpan = default;
ReadOnlySequence <byte> valueSequence = default;
try
{
switch (reader.TokenType)
{
// A new reader was created and has never been read,
// so we need to move to the first token.
// (or a reader has terminated and we're about to throw)
case JsonTokenType.None:
// Using a reader loop the caller has identified a property they wish to
// hydrate into a JsonDocument. Move to the value first.
case JsonTokenType.PropertyName:
{
if (!reader.Read())
{
if (shouldThrow)
{
ThrowHelper.ThrowJsonReaderException(
ref reader,
ExceptionResource.ExpectedJsonTokens);
}
reader = restore;
document = null;
return(false);
}
break;
}
}
switch (reader.TokenType)
{
// Any of the "value start" states are acceptable.
case JsonTokenType.StartObject:
case JsonTokenType.StartArray:
{
long startingOffset = reader.TokenStartIndex;
if (!reader.TrySkip())
{
if (shouldThrow)
{
ThrowHelper.ThrowJsonReaderException(
ref reader,
ExceptionResource.ExpectedJsonTokens);
}
reader = restore;
document = null;
return(false);
}
long totalLength = reader.BytesConsumed - startingOffset;
ReadOnlySequence <byte> sequence = reader.OriginalSequence;
if (sequence.IsEmpty)
{
valueSpan = reader.OriginalSpan.Slice(
checked ((int)startingOffset),
checked ((int)totalLength));
}
else
{
valueSequence = sequence.Slice(startingOffset, totalLength);
}
Debug.Assert(
reader.TokenType == JsonTokenType.EndObject ||
reader.TokenType == JsonTokenType.EndArray);
break;
}
case JsonTokenType.False:
case JsonTokenType.True:
case JsonTokenType.Null:
if (useArrayPools)
{
if (reader.HasValueSequence)
{
valueSequence = reader.ValueSequence;
}
else
{
valueSpan = reader.ValueSpan;
}
break;
}
document = CreateForLiteral(reader.TokenType);
return(true);
case JsonTokenType.Number:
{
if (reader.HasValueSequence)
{
valueSequence = reader.ValueSequence;
}
else
{
valueSpan = reader.ValueSpan;
}
break;
}
// String's ValueSequence/ValueSpan omits the quotes, we need them back.
case JsonTokenType.String:
{
ReadOnlySequence <byte> sequence = reader.OriginalSequence;
if (sequence.IsEmpty)
{
// Since the quoted string fit in a ReadOnlySpan originally
// the contents length plus the two quotes can't overflow.
int payloadLength = reader.ValueSpan.Length + 2;
Debug.Assert(payloadLength > 1);
ReadOnlySpan <byte> readerSpan = reader.OriginalSpan;
Debug.Assert(
readerSpan[(int)reader.TokenStartIndex] == (byte)'"',
$"Calculated span starts with {readerSpan[(int)reader.TokenStartIndex]}");
Debug.Assert(
readerSpan[(int)reader.TokenStartIndex + payloadLength - 1] == (byte)'"',
$"Calculated span ends with {readerSpan[(int)reader.TokenStartIndex + payloadLength - 1]}");
valueSpan = readerSpan.Slice((int)reader.TokenStartIndex, payloadLength);
}
else
{
long payloadLength = 2;
if (reader.HasValueSequence)
{
payloadLength += reader.ValueSequence.Length;
}
else
{
payloadLength += reader.ValueSpan.Length;
}
valueSequence = sequence.Slice(reader.TokenStartIndex, payloadLength);
Debug.Assert(
valueSequence.First.Span[0] == (byte)'"',
$"Calculated sequence starts with {valueSequence.First.Span[0]}");
Debug.Assert(
valueSequence.ToArray()[payloadLength - 1] == (byte)'"',
$"Calculated sequence ends with {valueSequence.ToArray()[payloadLength - 1]}");
}
break;
}
default:
{
if (shouldThrow)
{
// Default case would only hit if TokenType equals JsonTokenType.EndObject or JsonTokenType.EndArray in which case it would never be sequence
Debug.Assert(!reader.HasValueSequence);
byte displayByte = reader.ValueSpan[0];
ThrowHelper.ThrowJsonReaderException(
ref reader,
ExceptionResource.ExpectedStartOfValueNotFound,
displayByte);
}
reader = restore;
document = null;
return(false);
}
}
}
catch
{
reader = restore;
throw;
}
int length = valueSpan.IsEmpty ? checked ((int)valueSequence.Length) : valueSpan.Length;
if (useArrayPools)
{
byte[] rented = ArrayPool <byte> .Shared.Rent(length);
Span <byte> rentedSpan = rented.AsSpan(0, length);
try
{
if (valueSpan.IsEmpty)
{
valueSequence.CopyTo(rentedSpan);
}
else
{
valueSpan.CopyTo(rentedSpan);
}
document = Parse(rented.AsMemory(0, length), state.Options, rented);
}
catch
{
// This really shouldn't happen since the document was already checked
// for consistency by Skip. But if data mutations happened just after
// the calls to Read then the copy may not be valid.
rentedSpan.Clear();
ArrayPool <byte> .Shared.Return(rented);
throw;
}
}
else
{
byte[] owned;
if (valueSpan.IsEmpty)
{
owned = valueSequence.ToArray();
}
else
{
owned = valueSpan.ToArray();
}
document = ParseUnrented(owned, state.Options, reader.TokenType);
}
return(true);
}
public bool TryEncode(ReadOnlySequence <byte> sequence, out byte[] text, bool includePrefix = false)
{
text = Array.Empty <byte>();
if (sequence.IsEmpty)
{
return(true);
}
// Skip & count leading zeroes.
int zeroCount = 0;
foreach (var segment in sequence)
{
fixed(byte *p_segment_fixed = segment.Span)
{
byte *p_segment_start = p_segment_fixed;
byte *p_segment_end = p_segment_fixed + segment.Length;
while (p_segment_start != p_segment_end && *p_segment_start == 0)
{
zeroCount++;
p_segment_start++;
}
}
}
int length = 0;
// Allocate enough space in big-endian base58 representation.
int b58Length = ((int)(sequence.Length - zeroCount) * 138 / 100) + 1; // log(256) / log(58), rounded up.
using (var b58 = MemoryPool <byte> .Shared.Rent(b58Length))
{
BytesOperations.Zero(b58.Memory.Span);
fixed(byte *p_b58_fixed = b58.Memory.Span)
{
// Process the bytes.
foreach (var segment in sequence.Slice(zeroCount))
{
fixed(byte *p_segment_fixed = segment.Span)
{
byte *p_segment_start = p_segment_fixed;
byte *p_segment_end = p_segment_fixed + segment.Length;
while (p_segment_start != p_segment_end)
{
int carry = *p_segment_start;
int i = 0;
// Apply "b58 = b58 * 256 + ch".
byte *p_b58_start = (p_b58_fixed + b58Length) - 1;
byte *p_b58_end = p_b58_fixed - 1;
while (p_b58_start != p_b58_end && (carry != 0 || i < length))
{
carry += 256 * (*p_b58_start);
*p_b58_start = (byte)(carry % 58);
carry /= 58;
p_b58_start--;
i++;
}
length = i;
p_segment_start++;
}
}
}
{
byte *p_b58_start = p_b58_fixed + (b58Length - length);
byte *p_b58_end = p_b58_start + length;
// Skip leading zeroes in base58 result.
while (p_b58_start != p_b58_end && *p_b58_start == 0)
{
p_b58_start++;
}
var result = new byte[(includePrefix ? 1 : 0) + zeroCount + (p_b58_end - p_b58_start)];
fixed(byte *p_result_fixed = result)
{
byte *p_result_start = p_result_fixed;
if (includePrefix)
{
*p_result_start++ = (byte)'z';
}
for (int i = zeroCount - 1; i >= 0; i--)
{
*p_result_start++ = (byte)'1';
}
while (p_b58_start != p_b58_end)
{
*p_result_start++ = (byte)_base58Chars[*p_b58_start++];
}
}
text = result;
}
}
}
return(true);
}
public int Decode(ReadOnlySequence <byte> buffer)
{
DecoderHelper.ValidateSize(buffer.Length);
int bytesConsumed;
switch (currentState)
{
case State.ReadingSize:
bytesConsumed = sizeDecoder.Decode(buffer);
if (sizeDecoder.IsValueDecoded)
{
encodedSize = sizeDecoder.Value;
if (encodedSize > sizeQuota)
{
Exception quotaExceeded = OnSizeQuotaExceeded(encodedSize);
throw DiagnosticUtility.ExceptionUtility.ThrowHelperError(quotaExceeded);
}
if (encodedBytes == null || encodedBytes.Length < encodedSize)
{
encodedBytes = Fx.AllocateByteArray(encodedSize);
value = null;
}
currentState = State.ReadingBytes;
bytesNeeded = encodedSize;
}
break;
case State.ReadingBytes:
if (value != null && valueLengthInBytes == encodedSize && bytesNeeded == encodedSize &&
buffer.Length >= encodedSize && CompareBuffers(encodedBytes, buffer))
{
bytesConsumed = bytesNeeded;
OnComplete(value);
}
else
{
bytesConsumed = bytesNeeded;
if (buffer.Length < bytesNeeded)
{
bytesConsumed = (int)buffer.Length;
}
Span <byte> span = encodedBytes;
Span <byte> slicedBytes = span.Slice(encodedSize - bytesNeeded, bytesConsumed);
var tempBuffer = buffer.Slice(0, bytesConsumed);
tempBuffer.CopyTo(slicedBytes);
bytesNeeded -= bytesConsumed;
if (bytesNeeded == 0)
{
value = Encoding.UTF8.GetString(encodedBytes, 0, encodedSize);
valueLengthInBytes = encodedSize;
OnComplete(value);
}
}
break;
default:
throw DiagnosticUtility.ExceptionUtility.ThrowHelperError(new InvalidDataException(SR.InvalidDecoderStateMachine));
}
return(bytesConsumed);
}
public (int RequiredBytes, int HeaderLength, int BodyLength) TryRead(ref ReadOnlySequence <byte> input, out Message message)
{
if (input.Length < FramingLength)
{
message = default;
return(FramingLength, 0, 0);
}
Span <byte> lengthBytes = stackalloc byte[FramingLength];
input.Slice(input.Start, FramingLength).CopyTo(lengthBytes);
var headerLength = BinaryPrimitives.ReadInt32LittleEndian(lengthBytes);
var bodyLength = BinaryPrimitives.ReadInt32LittleEndian(lengthBytes.Slice(4));
// Check lengths
ThrowIfLengthsInvalid(headerLength, bodyLength);
var requiredBytes = FramingLength + headerLength + bodyLength;
if (input.Length < requiredBytes)
{
message = default;
return(requiredBytes, 0, 0);
}
try
{
// Decode header
var header = input.Slice(FramingLength, headerLength);
// Decode body
int bodyOffset = FramingLength + headerLength;
var body = input.Slice(bodyOffset, bodyLength);
// Build message
message = new();
if (header.IsSingleSegment)
{
var headersReader = Reader.Create(header.First.Span, _deserializationSession);
DeserializeFast(ref headersReader, message);
}
else
{
var headersReader = Reader.Create(header, _deserializationSession);
DeserializeFast(ref headersReader, message);
}
_deserializationSession.PartialReset();
// Body deserialization is more likely to fail than header deserialization.
// Separating the two allows for these kinds of errors to be propagated back to the caller.
if (body.IsSingleSegment)
{
message.BodyObject = _bodySerializer.Deserialize(body.First.Span, _deserializationSession);
}
else
{
message.BodyObject = _bodySerializer.Deserialize(body, _deserializationSession);
}
return(0, headerLength, bodyLength);
}
finally
{
input = input.Slice(requiredBytes);
_deserializationSession.PartialReset();
}
}
/// <summary>
/// Parse the sequence as Json
/// </summary>
/// <param name="stream">The sequence to parse</param>
/// <returns>A JsonElement that represents the sequence</returns>
/// <exception cref="Telefrek.Core.Json.InvalidJsonFormatException">If the buffer is not proper Json</exception>
internal static JsonElement ParseBuffer(ReadOnlySequence <byte> buffer)
{
var tree = new Stack <JsonElement>();
var current = (JsonElement)null;
var parent = (JsonElement)null;
var isName = false;
while (true)
{
if (buffer.Length > 0)
{
Trim(ref buffer);
switch (buffer.FirstSpan[0])
{
case START_ARRAY:
tree.Push(current);
current = new JsonArray();
buffer = buffer.Slice(1);
break;
case END_ARRAY:
parent = tree.Pop();
buffer = buffer.Slice(1);
if (parent != null)
{
if (parent.IsJsonArray())
{
parent.AsJsonArray().Items.Add(current);
}
else if (parent.IsJsonObject())
{
parent.AsJsonObject().Properties[parent.AsJsonObject().Properties.Count - 1].Value = current;
}
current = parent;
}
break;
case START_OBJECT:
tree.Push(current);
isName = true;
current = new JsonObject();
buffer = buffer.Slice(1);
break;
case END_OBJECT:
parent = tree.Pop();
isName = false;
buffer = buffer.Slice(1);
if (parent != null)
{
if (parent.IsJsonArray())
{
parent.AsJsonArray().Items.Add(current);
}
else if (parent.IsJsonObject())
{
parent.AsJsonObject().Properties[parent.AsJsonObject().Properties.Count - 1].Value = current;
isName = true;
}
current = parent;
}
break;
case COMMA:
// End of one object, properties should add here? child objects in array?
buffer = buffer.Slice(1);
isName = current != null && current.IsJsonObject();
break;
case COLON:
buffer = buffer.Slice(1);
isName = false;
break;
default:
// Check the current state for property reading
if (isName && TryReadString(ref buffer, out ReadOnlySequence <byte> name))
{
// Verify the state
if (current == null || !current.IsJsonObject())
{
throw new InvalidJsonFormatException("Attempted to read property without parent object");
}
// Create the property
var prop = new JsonProperty {
Name = Encoding.UTF8.GetString(name.ToArray()), Value = JsonNull.Instance
};
current.AsJsonObject().Properties.Add(prop);
isName = false;
}
// Try to read a primitive off the buffer
else if (TryReadPrimitive(ref buffer, out JsonElement element))
{
if (current == null)
{
current = element;
}
else if (current.IsJsonArray())
{
current.AsJsonArray().Items.Add(element);
}
else if (current.IsJsonObject())
{
var prop = current.AsJsonObject().Properties[current.AsJsonObject().Properties.Count - 1];
prop.Value = element;
}
else
{
throw new InvalidJsonFormatException("Attempted to add primitive to non object/array");
}
}
// This is an invalid buffer
else
{
throw new InvalidJsonFormatException("Failed to parse the buffer");
}
break;
}
}
// For multi-segment parsing of a read only sequence
private void ParseValuesSlow(
ref ReadOnlySequence <byte> buffer,
ref KeyValueAccumulator accumulator,
bool isFinalBlock)
{
var sequenceReader = new SequenceReader <byte>(buffer);
var consumed = sequenceReader.Position;
var equalsDelimiter = GetEqualsForEncoding();
var andDelimiter = GetAndForEncoding();
while (!sequenceReader.End)
{
// TODO seems there is a bug with TryReadTo (advancePastDelimiter: true). It isn't advancing past the delimiter on second read.
if (!sequenceReader.TryReadTo(out ReadOnlySequence <byte> key, equalsDelimiter, advancePastDelimiter: false) ||
!sequenceReader.IsNext(equalsDelimiter, true))
{
if (sequenceReader.Consumed > KeyLengthLimit)
{
ThrowKeyTooLargeException();
}
break;
}
if (key.Length > KeyLengthLimit)
{
ThrowKeyTooLargeException();
}
if (!sequenceReader.TryReadTo(out ReadOnlySequence <byte> value, andDelimiter, false) ||
!sequenceReader.IsNext(andDelimiter, true))
{
if (!isFinalBlock)
{
if (sequenceReader.Consumed - key.Length > ValueLengthLimit)
{
ThrowValueTooLargeException();
}
break;
}
value = buffer.Slice(sequenceReader.Position);
sequenceReader.Advance(value.Length);
}
if (value.Length > ValueLengthLimit)
{
ThrowValueTooLargeException();
}
// Need to call ToArray if the key/value spans multiple segments
var decodedKey = GetDecodedStringFromReadOnlySequence(key);
var decodedValue = GetDecodedStringFromReadOnlySequence(value);
AppendAndVerify(ref accumulator, decodedKey, decodedValue);
consumed = sequenceReader.Position;
}
buffer = buffer.Slice(consumed);
}
private ValueTask <bool> TryProcessPacketAsync(
ref ReadOnlySequence <byte> sequence,
CancellationToken cancellationToken = default)
{
var reader = new Utf8JsonReader(sequence);
VoiceGatewayPayload payload = default;
if (!Payload.TryReadOpcode(ref reader, out payload.Opcode))
{
return(new ValueTask <bool>(false));
}
switch (payload.Opcode)
{
case VoiceGatewayOpcode.Hello:
{
if (!Payload.TryReadHello(ref reader, out payload.Hello))
{
return(new ValueTask <bool>(false));
}
break;
}
case VoiceGatewayOpcode.Ready:
{
if (!Payload.TryReadReady(ref reader, out payload.Ready))
{
return(new ValueTask <bool>(false));
}
break;
}
case VoiceGatewayOpcode.SessionDescription:
{
if (!Payload.TryReadSessionDescription(ref reader,
_sessionEncryptionKey.Memory.Span.Slice(0,
SessionEncryptionKeyLength)))
{
return(new ValueTask <bool>(false));
}
break;
}
case VoiceGatewayOpcode.HeartbeatAck:
{
if (!Payload.TryReadHeartbeatAck(ref reader,
out payload.Nonce))
{
return(new ValueTask <bool>(false));
}
break;
}
case VoiceGatewayOpcode.Speaking:
{
return(new ValueTask <bool>(reader.TrySkip()));
}
default:
{
return(new ValueTask <bool>(reader.TrySkip()));
}
}
if (!reader.TryReadToken(JsonTokenType.EndObject))
{
return(new ValueTask <bool>(false));
}
sequence = sequence.Slice(reader.Position);
return(TryProcessOpcodeAsync(ref payload, cancellationToken));
}
// For multi-segment parsing of a read only sequence
private void ParseValuesSlow(
ref ReadOnlySequence <byte> buffer,
ref KeyValueAccumulator accumulator,
bool isFinalBlock)
{
var sequenceReader = new SequenceReader <byte>(buffer);
ReadOnlySequence <byte> keyValuePair;
var consumed = sequenceReader.Position;
var consumedBytes = default(long);
var equalsDelimiter = GetEqualsForEncoding();
var andDelimiter = GetAndForEncoding();
while (!sequenceReader.End)
{
if (!sequenceReader.TryReadTo(out keyValuePair, andDelimiter))
{
if (!isFinalBlock)
{
// Don't buffer indefinitely
if ((uint)(sequenceReader.Consumed - consumedBytes) > (uint)KeyLengthLimit + (uint)ValueLengthLimit)
{
ThrowKeyOrValueTooLargeException();
}
break;
}
// This must be the final key=value pair
keyValuePair = buffer.Slice(sequenceReader.Position);
sequenceReader.Advance(keyValuePair.Length);
}
if (keyValuePair.IsSingleSegment)
{
ParseFormValuesFast(keyValuePair.FirstSpan, ref accumulator, isFinalBlock: true, out var segmentConsumed);
Debug.Assert(segmentConsumed == keyValuePair.FirstSpan.Length);
consumedBytes = sequenceReader.Consumed;
consumed = sequenceReader.Position;
continue;
}
var keyValueReader = new SequenceReader <byte>(keyValuePair);
ReadOnlySequence <byte> value;
if (keyValueReader.TryReadTo(out var key, equalsDelimiter))
{
if (key.Length > KeyLengthLimit)
{
ThrowKeyTooLargeException();
}
value = keyValuePair.Slice(keyValueReader.Position);
if (value.Length > ValueLengthLimit)
{
ThrowValueTooLargeException();
}
}
else
{
// Too long for the whole segment to be a key.
if (keyValuePair.Length > KeyLengthLimit)
{
ThrowKeyTooLargeException();
}
// There is no more data, this segment must be "key" with no equals or value.
key = keyValuePair;
value = default;
}
var decodedKey = GetDecodedStringFromReadOnlySequence(key);
var decodedValue = GetDecodedStringFromReadOnlySequence(value);
AppendAndVerify(ref accumulator, decodedKey, decodedValue);
consumedBytes = sequenceReader.Consumed;
consumed = sequenceReader.Position;
}
buffer = buffer.Slice(consumed);
}
protected override int ParseBodySizeFromHeader(ref ReadOnlySequence<byte> buffer)
{
return BitConverter.ToInt32(buffer.Slice(0, 4).FirstSpan);
}
public static bool TryParseMessage(ref ReadOnlySequence <byte> buffer, out ReadOnlySequence <byte> payload)
{
if (buffer.IsEmpty)
{
payload = default;
return(false);
}
// The payload starts with a length prefix encoded as a VarInt. VarInts use the most significant bit
// as a marker whether the byte is the last byte of the VarInt or if it spans to the next byte. Bytes
// appear in the reverse order - i.e. the first byte contains the least significant bits of the value
// Examples:
// VarInt: 0x35 - %00110101 - the most significant bit is 0 so the value is %x0110101 i.e. 0x35 (53)
// VarInt: 0x80 0x25 - %10000000 %00101001 - the most significant bit of the first byte is 1 so the
// remaining bits (%x0000000) are the lowest bits of the value. The most significant bit of the second
// byte is 0 meaning this is last byte of the VarInt. The actual value bits (%x0101001) need to be
// prepended to the bits we already read so the values is %01010010000000 i.e. 0x1480 (5248)
// We support paylads up to 2GB so the biggest number we support is 7fffffff which when encoded as
// VarInt is 0xFF 0xFF 0xFF 0xFF 0x07 - hence the maximum length prefix is 5 bytes.
var length = 0U;
var numBytes = 0;
var lengthPrefixBuffer = buffer.Slice(0, Math.Min(MaxLengthPrefixSize, buffer.Length));
var span = GetSpan(lengthPrefixBuffer);
byte byteRead;
do
{
byteRead = span[numBytes];
length = length | (((uint)(byteRead & 0x7f)) << (numBytes * 7));
numBytes++;
}while (numBytes < lengthPrefixBuffer.Length && ((byteRead & 0x80) != 0));
// size bytes are missing
if ((byteRead & 0x80) != 0 && (numBytes < MaxLengthPrefixSize))
{
payload = default;
return(false);
}
if ((byteRead & 0x80) != 0 || (numBytes == MaxLengthPrefixSize && byteRead > 7))
{
throw new FormatException("Messages over 2GB in size are not supported.");
}
// We don't have enough data
if (buffer.Length < length + numBytes)
{
payload = default;
return(false);
}
// Get the payload
payload = buffer.Slice(numBytes, (int)length);
// Skip the payload
buffer = buffer.Slice(numBytes + (int)length);
return(true);
}
/// <summary>
/// Gets the packet header from the current packet.
/// </summary>
public int GetHeader(ReadOnlySequence <byte> buffer) => RecvCipher
.Handshaken
? 4 + MapleCipher.GetPacketLength(buffer.Slice(0, 4).ToSpan())
: 2 + BitConverter.ToUInt16(buffer.Slice(0, 2).ToSpan());
protected override TextPackageInfo DecodePackage(ReadOnlySequence <byte> buffer)
{
return(new TextPackageInfo {
Text = buffer.Slice(4).GetString(Encoding.UTF8)
});
}
public int Send(ReadOnlySequence <byte> span, object option = null)
{
if (CheckDispose())
{
//检查释放
return(-4);
}
if (mss <= 0)
{
throw new InvalidOperationException($" mss <= 0 ");
}
if (span.Length == 0)
{
return(-1);
}
var offset = 0;
int count;
#region append to previous segment in streaming mode (if possible)
/// 基于线程安全和数据结构的等原因,移除了追加数据到最后一个包行为。
#endregion
#region fragment
if (span.Length <= mss)
{
count = 1;
}
else
{
count = (int)(span.Length + mss - 1) / (int)mss;
}
if (count > IKCP_WND_RCV)
{
return(-2);
}
if (count == 0)
{
count = 1;
}
for (var i = 0; i < count; i++)
{
int size;
if (span.Length - offset > mss)
{
size = (int)mss;
}
else
{
size = (int)span.Length - offset;
}
var seg = SegmentManager.Alloc(size);
span.Slice(offset, size).CopyTo(seg.data);
offset += size;
seg.frg = (byte)(count - i - 1);
snd_queue.Enqueue(seg);
}
#endregion
return(0);
}
private static void ReadValueCore(JsonSerializerOptions options, ref Utf8JsonReader reader, ref ReadStack readStack)
{
JsonReaderState state = reader.CurrentState;
CheckSupportedOptions(state.Options, nameof(reader));
// Value copy to overwrite the ref on an exception and undo the destructive reads.
Utf8JsonReader restore = reader;
ReadOnlySpan <byte> valueSpan = default;
ReadOnlySequence <byte> valueSequence = default;
try
{
switch (reader.TokenType)
{
// A new reader was created and has never been read,
// so we need to move to the first token.
// (or a reader has terminated and we're about to throw)
case JsonTokenType.None:
// Using a reader loop the caller has identified a property they wish to
// hydrate into a JsonDocument. Move to the value first.
case JsonTokenType.PropertyName:
{
if (!reader.Read())
{
ThrowHelper.ThrowJsonReaderException(ref reader, ExceptionResource.ExpectedOneCompleteToken);
}
break;
}
}
switch (reader.TokenType)
{
// Any of the "value start" states are acceptable.
case JsonTokenType.StartObject:
case JsonTokenType.StartArray:
{
long startingOffset = reader.TokenStartIndex;
if (!reader.TrySkip())
{
ThrowHelper.ThrowJsonReaderException(ref reader, ExceptionResource.NotEnoughData);
}
long totalLength = reader.BytesConsumed - startingOffset;
ReadOnlySequence <byte> sequence = reader.OriginalSequence;
if (sequence.IsEmpty)
{
valueSpan = reader.OriginalSpan.Slice(
checked ((int)startingOffset),
checked ((int)totalLength));
}
else
{
valueSequence = sequence.Slice(startingOffset, totalLength);
}
Debug.Assert(
reader.TokenType == JsonTokenType.EndObject ||
reader.TokenType == JsonTokenType.EndArray);
break;
}
// Single-token values
case JsonTokenType.Number:
case JsonTokenType.True:
case JsonTokenType.False:
case JsonTokenType.Null:
{
if (reader.HasValueSequence)
{
valueSequence = reader.ValueSequence;
}
else
{
valueSpan = reader.ValueSpan;
}
break;
}
// String's ValueSequence/ValueSpan omits the quotes, we need them back.
case JsonTokenType.String:
{
ReadOnlySequence <byte> sequence = reader.OriginalSequence;
if (sequence.IsEmpty)
{
// Since the quoted string fit in a ReadOnlySpan originally
// the contents length plus the two quotes can't overflow.
int payloadLength = reader.ValueSpan.Length + 2;
Debug.Assert(payloadLength > 1);
ReadOnlySpan <byte> readerSpan = reader.OriginalSpan;
Debug.Assert(
readerSpan[(int)reader.TokenStartIndex] == (byte)'"',
$"Calculated span starts with {readerSpan[(int)reader.TokenStartIndex]}");
Debug.Assert(
readerSpan[(int)reader.TokenStartIndex + payloadLength - 1] == (byte)'"',
$"Calculated span ends with {readerSpan[(int)reader.TokenStartIndex + payloadLength - 1]}");
valueSpan = readerSpan.Slice((int)reader.TokenStartIndex, payloadLength);
}
else
{
long payloadLength = 2;
if (reader.HasValueSequence)
{
payloadLength += reader.ValueSequence.Length;
}
else
{
payloadLength += reader.ValueSpan.Length;
}
valueSequence = sequence.Slice(reader.TokenStartIndex, payloadLength);
Debug.Assert(
valueSequence.First.Span[0] == (byte)'"',
$"Calculated sequence starts with {valueSequence.First.Span[0]}");
Debug.Assert(
valueSequence.ToArray()[payloadLength - 1] == (byte)'"',
$"Calculated sequence ends with {valueSequence.ToArray()[payloadLength - 1]}");
}
break;
}
default:
{
byte displayByte;
if (reader.HasValueSequence)
{
displayByte = reader.ValueSequence.First.Span[0];
}
else
{
displayByte = reader.ValueSpan[0];
}
ThrowHelper.ThrowJsonReaderException(
ref reader,
ExceptionResource.ExpectedStartOfValueNotFound,
displayByte);
break;
}
}
}
catch (JsonReaderException ex)
{
reader = restore;
// Re-throw with Path information.
ThrowHelper.ReThrowWithPath(readStack, ex);
}
int length = valueSpan.IsEmpty ? checked ((int)valueSequence.Length) : valueSpan.Length;
byte[] rented = ArrayPool <byte> .Shared.Rent(length);
Span <byte> rentedSpan = rented.AsSpan(0, length);
try
{
if (valueSpan.IsEmpty)
{
valueSequence.CopyTo(rentedSpan);
}
else
{
valueSpan.CopyTo(rentedSpan);
}
JsonReaderOptions originalReaderOptions = state.Options;
var newReader = new Utf8JsonReader(rentedSpan, originalReaderOptions);
ReadCore(options, ref newReader, ref readStack);
// The reader should have thrown if we have remaining bytes.
Debug.Assert(newReader.BytesConsumed == length);
}
catch (JsonException)
{
reader = restore;
throw;
}
finally
{
rentedSpan.Clear();
ArrayPool <byte> .Shared.Return(rented);
}
}
private TResult Read <TResult, TParser>(TParser parser)
where TParser : struct, IBufferReader <TResult>
{
parser.Append <TResult, TParser>(sequence.Slice(position), out position);
return(parser.RemainingBytes == 0 ? parser.Complete() : throw new EndOfStreamException());
}
private static bool HasValidChecksum(ReadOnlySequence <byte> input) => input.ReadUInt32(Constants.MagicNumber.Length, true) == Crc32C.Calculate(input.Slice(Constants.MetadataSizeOffset));
public OnlinePackageInfo Decode(ref ReadOnlySequence <byte> buffer, object context)
{
if (buffer.Length < HEADER_LENGTH)
{
throw new Exception($"proto error:head lenth less than { HEADER_LENGTH }");
}
try
{
var reader = new SequenceReader <byte>(buffer);
var packgae = new OnlinePackageInfo();
reader.Advance(1);
if (reader.TryReadBigEndian(out short key))
{
packgae.Key = key;
}
reader.Advance(1);
if (reader.TryRead(out byte version))
{
packgae.Version = version;
}
packgae.RequestID = new Guid(reader.CurrentSpan.Slice(6, 16));
reader.Advance(16);
if (reader.TryReadLittleEndian(out int crcCode))
{
packgae.CrcCode = crcCode;
}
if (reader.TryReadLittleEndian(out int bodyLength))
{
packgae.BodyLength = bodyLength;
}
if (buffer.Length != packgae.BodyLength + HEADER_LENGTH)
{
throw new Exception($"Package error:length{ packgae.BodyLength },active length{ buffer.Length - HEADER_LENGTH } ");
}
if (packgae.BodyLength > 0)
{
if (OnlinePackageDecoderFactory.GetFactory(packgae.Key, out IDecoderFactory onlinePackageFactory))
{
var sclie = buffer.Slice(HEADER_LENGTH, packgae.BodyLength);
packgae.Object = onlinePackageFactory.Create(ref sclie);
}
else
{
_logger.LogError("Not find the command");
}
}
return(packgae);
}
catch (Exception ex)
{
_logger.LogError(ex.Message);
return(default);
protected bool Read(ReadOnlySequence <byte> readableBuffer, PipeWriter writableBuffer, out SequencePosition consumed, out SequencePosition examined)
{
consumed = default;
examined = default;
while (_mode < Mode.Trailer)
{
if (_mode == Mode.Prefix)
{
ParseChunkedPrefix(readableBuffer, out consumed, out examined);
if (_mode == Mode.Prefix)
{
return(false);
}
readableBuffer = readableBuffer.Slice(consumed);
}
if (_mode == Mode.Extension)
{
ParseExtension(readableBuffer, out consumed, out examined);
if (_mode == Mode.Extension)
{
return(false);
}
readableBuffer = readableBuffer.Slice(consumed);
}
if (_mode == Mode.Data)
{
ReadChunkedData(readableBuffer, writableBuffer, out consumed, out examined);
if (_mode == Mode.Data)
{
return(false);
}
readableBuffer = readableBuffer.Slice(consumed);
}
if (_mode == Mode.Suffix)
{
ParseChunkedSuffix(readableBuffer, out consumed, out examined);
if (_mode == Mode.Suffix)
{
return(false);
}
readableBuffer = readableBuffer.Slice(consumed);
}
}
// Chunks finished, parse trailers
if (_mode == Mode.Trailer)
{
ParseChunkedTrailer(readableBuffer, out consumed, out examined);
if (_mode == Mode.Trailer)
{
return(false);
}
readableBuffer = readableBuffer.Slice(consumed);
}
// _consumedBytes aren't tracked for trailer headers, since headers have separate limits.
if (_mode == Mode.TrailerHeaders)
{
if (_context.TakeMessageHeaders(readableBuffer, out consumed, out examined))
{
_mode = Mode.Complete;
}
}
return(_mode == Mode.Complete);
}
private static (int keySize, int keyCount, int bytesRead)? TryReadHeader(ReadOnlySequence <byte> sequence)
{
Span <ulong> headerValues = stackalloc ulong[3];
var headerBytes = MemoryMarshal.AsBytes(headerValues);
if (sequence.Length < headerBytes.Length)
{
return(null);
}
sequence.Slice(0, headerBytes.Length).CopyTo(headerBytes);
// https://github.com/ClickHouse/ClickHouse/blob/master/src/DataTypes/DataTypeLowCardinality.cpp
// Dictionary is written as number N and N keys after them.
// Dictionary can be shared for continuous range of granules, so some marks may point to the same position.
// Shared dictionary is stored in state and is read once.
//
// SharedDictionariesWithAdditionalKeys = 1,
if (headerValues[0] != 1)
{
throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, $"Internal error. Unexpected dictionary version: {headerBytes[0]}.");
}
var keySizeCode = unchecked ((byte)headerValues[1]);
int keySize;
switch (keySizeCode)
{
case 0:
keySize = 1;
break;
case 1:
keySize = 2;
break;
case 3:
keySize = 4;
break;
case 4:
throw new NotSupportedException("64-bit keys are not supported.");
default:
throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, $"Internal error. Unexpected size of a key: {keySizeCode}.");
}
// There are several control flags, but the client always receives 0x2|0x4
//
// https://github.com/ClickHouse/ClickHouse/blob/master/src/DataTypes/DataTypeLowCardinality.cpp
// 0x1 Need to read dictionary if it wasn't.
// 0x2 Need to read additional keys. Additional keys are stored before indexes as value N and N keys after them.
// 0x4 Need to update dictionary. It means that previous granule has different dictionary.
var flags = headerValues[1] >> 8;
if (flags != (0x2 | 0x4))
{
throw new NotSupportedException("Received combination of flags is not supported.");
}
var keyCount = (int)headerValues[2];
return(keySize, keyCount, headerBytes.Length);
}