private static void CheckLastTagWas(ref ParserInternalState state, uint expectedTag) { if (state.lastTag != expectedTag) { throw InvalidProtocolBufferException.InvalidEndTag(); } }
/// <summary> /// Reads and discards <paramref name="size"/> bytes. /// </summary> /// <exception cref="InvalidProtocolBufferException">the end of the stream /// or the current limit was reached</exception> public static void SkipRawBytes(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state, int size) { if (size < 0) { throw InvalidProtocolBufferException.NegativeSize(); } ValidateCurrentLimit(ref buffer, ref state, size); if (size <= state.bufferSize - state.bufferPos) { // We have all the bytes we need already. state.bufferPos += size; } else { // Skipping more bytes than are in the buffer. First skip what we have. int pos = state.bufferSize - state.bufferPos; state.bufferPos = state.bufferSize; // TODO: If our segmented buffer is backed by a Stream that is seekable, we could skip the bytes more efficiently // by simply updating stream's Position property. This used to be supported in the past, but the support was dropped // because it would make the segmentedBufferHelper more complex. Support can be reintroduced if needed. state.segmentedBufferHelper.RefillBuffer(ref buffer, ref state, true); while (size - pos > state.bufferSize) { pos += state.bufferSize; state.bufferPos = state.bufferSize; state.segmentedBufferHelper.RefillBuffer(ref buffer, ref state, true); } state.bufferPos = size - pos; } }
public static void SkipLastField(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state) { if (state.lastTag == 0) { throw new InvalidOperationException("SkipLastField cannot be called at the end of a stream"); } switch (WireFormat.GetTagWireType(state.lastTag)) { case WireFormat.WireType.StartGroup: SkipGroup(ref buffer, ref state, state.lastTag); break; case WireFormat.WireType.EndGroup: throw new InvalidProtocolBufferException( "SkipLastField called on an end-group tag, indicating that the corresponding start-group was missing"); case WireFormat.WireType.Fixed32: ParsingPrimitives.ParseRawLittleEndian32(ref buffer, ref state); break; case WireFormat.WireType.Fixed64: ParsingPrimitives.ParseRawLittleEndian64(ref buffer, ref state); break; case WireFormat.WireType.LengthDelimited: var length = ParsingPrimitives.ParseLength(ref buffer, ref state); ParsingPrimitives.SkipRawBytes(ref buffer, ref state, length); break; case WireFormat.WireType.Varint: ParsingPrimitives.ParseRawVarint32(ref buffer, ref state); break; } }
/// <summary> /// Parses a raw varint. /// </summary> public static ulong ParseRawVarint64(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state) { if (state.bufferPos + 10 > state.bufferSize) { return(ParseRawVarint64SlowPath(ref buffer, ref state)); } ulong result = buffer[state.bufferPos++]; if (result < 128) { return(result); } result &= 0x7f; int shift = 7; do { byte b = buffer[state.bufferPos++]; result |= (ulong)(b & 0x7F) << shift; if (b < 0x80) { return(result); } shift += 7; }while (shift < 64); throw InvalidProtocolBufferException.MalformedVarint(); }
/// <summary> /// Verifies that the last call to ReadTag() returned tag 0 - in other words, /// we've reached the end of the stream when we expected to. /// </summary> /// <exception cref="InvalidProtocolBufferException">The /// tag read was not the one specified</exception> public static void CheckReadEndOfStreamTag(ref ParserInternalState state) { if (state.lastTag != 0) { throw InvalidProtocolBufferException.MoreDataAvailable(); } }
internal static ulong?ReadUInt64WrapperSlow(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state) { // field=1, type=varint = tag of 8 const int expectedTag = 8; int length = ParsingPrimitives.ParseLength(ref buffer, ref state); if (length == 0) { return(0L); } int finalBufferPos = state.totalBytesRetired + state.bufferPos + length; ulong result = 0L; do { if (ParsingPrimitives.ParseTag(ref buffer, ref state) == expectedTag) { result = ParsingPrimitives.ParseRawVarint64(ref buffer, ref state); } else { ParsingPrimitivesMessages.SkipLastField(ref buffer, ref state); } }while (state.totalBytesRetired + state.bufferPos < finalBufferPos); return(result); }
internal static float?ReadFloatWrapperSlow(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state) { int length = ParsingPrimitives.ParseLength(ref buffer, ref state); if (length == 0) { return(0F); } int finalBufferPos = state.totalBytesRetired + state.bufferPos + length; float result = 0F; do { // field=1, type=32-bit = tag of 13 if (ParsingPrimitives.ParseTag(ref buffer, ref state) == 13) { result = ParsingPrimitives.ParseFloat(ref buffer, ref state); } else { ParsingPrimitivesMessages.SkipLastField(ref buffer, ref state); } }while (state.totalBytesRetired + state.bufferPos < finalBufferPos); return(result); }
internal static double?ReadDoubleWrapperSlow(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state) { int length = ParsingPrimitives.ParseLength(ref buffer, ref state); if (length == 0) { return(0D); } int finalBufferPos = state.totalBytesRetired + state.bufferPos + length; double result = 0D; do { // field=1, type=64-bit = tag of 9 if (ParsingPrimitives.ParseTag(ref buffer, ref state) == 9) { result = ParsingPrimitives.ParseDouble(ref buffer, ref state); } else { ParsingPrimitivesMessages.SkipLastField(ref buffer, ref state); } }while (state.totalBytesRetired + state.bufferPos < finalBufferPos); return(result); }
internal static uint?ReadUInt32WrapperSlow(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state) { int length = ParsingPrimitives.ParseLength(ref buffer, ref state); if (length == 0) { return(0); } int finalBufferPos = state.totalBytesRetired + state.bufferPos + length; uint result = 0; do { // field=1, type=varint = tag of 8 if (ParsingPrimitives.ParseTag(ref buffer, ref state) == 8) { result = ParsingPrimitives.ParseRawVarint32(ref buffer, ref state); } else { ParsingPrimitivesMessages.SkipLastField(ref buffer, ref state); } }while (state.totalBytesRetired + state.bufferPos < finalBufferPos); return(result); }
private static void CheckCurrentBufferIsEmpty(ref ParserInternalState state) { if (state.bufferPos < state.bufferSize) { throw new InvalidOperationException("RefillBuffer() called when buffer wasn't empty."); } }
public static string ReadRawString(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state, int length) { // No need to read any data for an empty string. if (length == 0) { return(string.Empty); } if (length < 0) { throw InvalidProtocolBufferException.NegativeSize(); } #if GOOGLE_PROTOBUF_SUPPORT_FAST_STRING if (length <= state.bufferSize - state.bufferPos) { // Fast path: all bytes to decode appear in the same span. ReadOnlySpan <byte> data = buffer.Slice(state.bufferPos, length); string value; unsafe { fixed(byte *sourceBytes = &MemoryMarshal.GetReference(data)) { value = WritingPrimitives.Utf8Encoding.GetString(sourceBytes, length); } } state.bufferPos += length; return(value); } #endif return(ReadStringSlow(ref buffer, ref state, length)); }
private static byte[] ReadRawBytesSlow(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state, int size) { ValidateCurrentLimit(ref buffer, ref state, size); if ((!state.segmentedBufferHelper.TotalLength.HasValue && size < buffer.Length) || IsDataAvailableInSource(ref state, size)) { // Reading more bytes than are in the buffer, but not an excessive number // of bytes. We can safely allocate the resulting array ahead of time. byte[] bytes = new byte[size]; ReadRawBytesIntoSpan(ref buffer, ref state, size, bytes); return(bytes); } else { // The size is very large. For security reasons, we can't allocate the // entire byte array yet. The size comes directly from the input, so a // maliciously-crafted message could provide a bogus very large size in // order to trick the app into allocating a lot of memory. We avoid this // by allocating and reading only a small chunk at a time, so that the // malicious message must actually *be* extremely large to cause // problems. Meanwhile, we limit the allowed size of a message elsewhere. List <byte[]> chunks = new List <byte[]>(); int pos = state.bufferSize - state.bufferPos; byte[] firstChunk = new byte[pos]; buffer.Slice(state.bufferPos, pos).CopyTo(firstChunk); chunks.Add(firstChunk); state.bufferPos = state.bufferSize; // Read all the rest of the bytes we need. int sizeLeft = size - pos; while (sizeLeft > 0) { state.segmentedBufferHelper.RefillBuffer(ref buffer, ref state, true); byte[] chunk = new byte[Math.Min(sizeLeft, state.bufferSize)]; buffer.Slice(0, chunk.Length) .CopyTo(chunk); state.bufferPos += chunk.Length; sizeLeft -= chunk.Length; chunks.Add(chunk); } // OK, got everything. Now concatenate it all into one buffer. byte[] bytes = new byte[size]; int newPos = 0; foreach (byte[] chunk in chunks) { Buffer.BlockCopy(chunk, 0, bytes, newPos, chunk.Length); newPos += chunk.Length; } // Done. return(bytes); } }
private static byte ReadRawByte(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state) { if (state.bufferPos == state.bufferSize) { state.segmentedBufferHelper.RefillBuffer(ref buffer, ref state, true); } return(buffer[state.bufferPos++]); }
/// <summary> /// Parses a raw Varint. If larger than 32 bits, discard the upper bits. /// This method is optimised for the case where we've got lots of data in the buffer. /// That means we can check the size just once, then just read directly from the buffer /// without constant rechecking of the buffer length. /// </summary> public static uint ParseRawVarint32(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state) { if (state.bufferPos + 5 > state.bufferSize) { return(ParseRawVarint32SlowPath(ref buffer, ref state)); } int tmp = buffer[state.bufferPos++]; if (tmp < 128) { return((uint)tmp); } int result = tmp & 0x7f; if ((tmp = buffer[state.bufferPos++]) < 128) { result |= tmp << 7; } else { result |= (tmp & 0x7f) << 7; if ((tmp = buffer[state.bufferPos++]) < 128) { result |= tmp << 14; } else { result |= (tmp & 0x7f) << 14; if ((tmp = buffer[state.bufferPos++]) < 128) { result |= tmp << 21; } else { result |= (tmp & 0x7f) << 21; result |= (tmp = buffer[state.bufferPos++]) << 28; if (tmp >= 128) { // Discard upper 32 bits. // Note that this has to use ReadRawByte() as we only ensure we've // got at least 5 bytes at the start of the method. This lets us // use the fast path in more cases, and we rarely hit this section of code. for (int i = 0; i < 5; i++) { if (ReadRawByte(ref buffer, ref state) < 128) { return((uint)result); } } throw InvalidProtocolBufferException.MalformedVarint(); } } } } return((uint)result); }
/// <summary> /// Peeks at the next tag in the stream. If it matches <paramref name="tag"/>, /// the tag is consumed and the method returns <c>true</c>; otherwise, the /// stream is left in the original position and the method returns <c>false</c>. /// </summary> public static bool MaybeConsumeTag(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state, uint tag) { if (PeekTag(ref buffer, ref state) == tag) { state.hasNextTag = false; return(true); } return(false); }
/// <summary> /// Returns whether or not all the data before the limit has been read. /// </summary> /// <returns></returns> public static bool IsReachedLimit(ref ParserInternalState state) { if (state.currentLimit == int.MaxValue) { return(false); } int currentAbsolutePosition = state.totalBytesRetired + state.bufferPos; return(currentAbsolutePosition >= state.currentLimit); }
/// <summary> /// Checks whether there is known data available of the specified size remaining to parse. /// When parsing from a Stream this can return false because we have no knowledge of the amount /// of data remaining in the stream until it is read. /// </summary> public static bool IsDataAvailable(ref ParserInternalState state, int size) { // Data fits in remaining buffer if (size <= state.bufferSize - state.bufferPos) { return(true); } return(IsDataAvailableInSource(ref state, size)); }
/// <summary> /// Validates that the specified size doesn't exceed the current limit. If it does then remaining bytes /// are skipped and an error is thrown. /// </summary> private static void ValidateCurrentLimit(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state, int size) { if (state.totalBytesRetired + state.bufferPos + size > state.currentLimit) { // Read to the end of the stream (up to the current limit) anyway. SkipRawBytes(ref buffer, ref state, state.currentLimit - state.totalBytesRetired - state.bufferPos); // Then fail. throw InvalidProtocolBufferException.TruncatedMessage(); } }
/// <summary> /// Parses the next tag. /// If the end of logical stream was reached, an invalid tag of 0 is returned. /// </summary> public static uint ParseTag(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state) { // The "nextTag" logic is there only as an optimization for reading non-packed repeated / map // fields and is strictly speaking not necessary. // TODO(jtattermusch): look into simplifying the ParseTag logic. if (state.hasNextTag) { state.lastTag = state.nextTag; state.hasNextTag = false; return(state.lastTag); } // Optimize for the incredibly common case of having at least two bytes left in the buffer, // and those two bytes being enough to get the tag. This will be true for fields up to 4095. if (state.bufferPos + 2 <= state.bufferSize) { int tmp = buffer[state.bufferPos++]; if (tmp < 128) { state.lastTag = (uint)tmp; } else { int result = tmp & 0x7f; if ((tmp = buffer[state.bufferPos++]) < 128) { result |= tmp << 7; state.lastTag = (uint)result; } else { // Nope, rewind and go the potentially slow route. state.bufferPos -= 2; state.lastTag = ParsingPrimitives.ParseRawVarint32(ref buffer, ref state); } } } else { if (SegmentedBufferHelper.IsAtEnd(ref buffer, ref state)) { state.lastTag = 0; return(0); } state.lastTag = ParsingPrimitives.ParseRawVarint32(ref buffer, ref state); } if (WireFormat.GetTagFieldNumber(state.lastTag) == 0) { // If we actually read a tag with a field of 0, that's not a valid tag. throw InvalidProtocolBufferException.InvalidTag(); } return(state.lastTag); }
public bool RefillBuffer(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state, bool mustSucceed) { if (codedInputStream != null) { return(RefillFromCodedInputStream(ref buffer, ref state, mustSucceed)); } else { return(RefillFromReadOnlySequence(ref buffer, ref state, mustSucceed)); } }
internal static void Initialize(ReadOnlySpan <byte> buffer, out ParseContext ctx) { ParserInternalState state = default; state.sizeLimit = DefaultSizeLimit; state.recursionLimit = DefaultRecursionLimit; state.currentLimit = int.MaxValue; state.bufferSize = buffer.Length; Initialize(buffer, ref state, out ctx); }
private bool RefillFromReadOnlySequence(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state, bool mustSucceed) { CheckCurrentBufferIsEmpty(ref state); if (state.totalBytesRetired + state.bufferSize == state.currentLimit) { // Oops, we hit a limit. if (mustSucceed) { throw InvalidProtocolBufferException.TruncatedMessage(); } else { return(false); } } state.totalBytesRetired += state.bufferSize; state.bufferPos = 0; state.bufferSize = 0; while (readOnlySequenceEnumerator.MoveNext()) { buffer = readOnlySequenceEnumerator.Current.Span; state.bufferSize = buffer.Length; if (buffer.Length != 0) { break; } } if (state.bufferSize == 0) { if (mustSucceed) { throw InvalidProtocolBufferException.TruncatedMessage(); } else { return(false); } } else { RecomputeBufferSizeAfterLimit(ref state); int totalBytesRead = state.totalBytesRetired + state.bufferSize + state.bufferSizeAfterLimit; if (totalBytesRead < 0 || totalBytesRead > state.sizeLimit) { throw InvalidProtocolBufferException.SizeLimitExceeded(); } return(true); } }
/// <summary> /// Parses a 64-bit little-endian integer. /// </summary> public static ulong ParseRawLittleEndian64(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state) { const int length = sizeof(ulong); if (state.bufferPos + length > state.bufferSize) { return(ParseRawLittleEndian64SlowPath(ref buffer, ref state)); } ulong result = BinaryPrimitives.ReadUInt64LittleEndian(buffer.Slice(state.bufferPos, length)); state.bufferPos += length; return(result); }
private bool RefillFromCodedInputStream(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state, bool mustSucceed) { CheckCurrentBufferIsEmpty(ref state); if (state.totalBytesRetired + state.bufferSize == state.currentLimit) { // Oops, we hit a limit. if (mustSucceed) { throw InvalidProtocolBufferException.TruncatedMessage(); } else { return(false); } } Stream input = codedInputStream.InternalInputStream; state.totalBytesRetired += state.bufferSize; state.bufferPos = 0; state.bufferSize = (input == null) ? 0 : input.Read(codedInputStream.InternalBuffer, 0, buffer.Length); if (state.bufferSize < 0) { throw new InvalidOperationException("Stream.Read returned a negative count"); } if (state.bufferSize == 0) { if (mustSucceed) { throw InvalidProtocolBufferException.TruncatedMessage(); } else { return(false); } } else { RecomputeBufferSizeAfterLimit(ref state); int totalBytesRead = state.totalBytesRetired + state.bufferSize + state.bufferSizeAfterLimit; if (totalBytesRead < 0 || totalBytesRead > state.sizeLimit) { throw InvalidProtocolBufferException.SizeLimitExceeded(); } return(true); } }
/// <summary> /// Parses a float value. /// </summary> public static float ParseFloat(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state) { const int length = sizeof(float); if (!BitConverter.IsLittleEndian || state.bufferPos + length > state.bufferSize) { return(ParseFloatSlow(ref buffer, ref state)); } // ReadUnaligned uses processor architecture for endianness. float result = Unsafe.ReadUnaligned <float>(ref MemoryMarshal.GetReference(buffer.Slice(state.bufferPos, length))); state.bufferPos += length; return(result); }
/// <summary> /// Parses a double value. /// </summary> public static double ParseDouble(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state) { const int length = sizeof(double); if (!BitConverter.IsLittleEndian || state.bufferPos + length > state.bufferSize) { return(BitConverter.Int64BitsToDouble((long)ParseRawLittleEndian64(ref buffer, ref state))); } // ReadUnaligned uses processor architecture for endianness. double result = Unsafe.ReadUnaligned <double>(ref MemoryMarshal.GetReference(buffer.Slice(state.bufferPos, length))); state.bufferPos += length; return(result); }
/// <summary> /// Peeks at the next field tag. This is like calling <see cref="ParseTag"/>, but the /// tag is not consumed. (So a subsequent call to <see cref="ParseTag"/> will return the /// same value.) /// </summary> public static uint PeekTag(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state) { if (state.hasNextTag) { return(state.nextTag); } uint savedLast = state.lastTag; state.nextTag = ParseTag(ref buffer, ref state); state.hasNextTag = true; state.lastTag = savedLast; // Undo the side effect of ReadTag return(state.nextTag); }
private static uint ParseRawVarint32SlowPath(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state) { int tmp = ReadRawByte(ref buffer, ref state); if (tmp < 128) { return((uint)tmp); } int result = tmp & 0x7f; if ((tmp = ReadRawByte(ref buffer, ref state)) < 128) { result |= tmp << 7; } else { result |= (tmp & 0x7f) << 7; if ((tmp = ReadRawByte(ref buffer, ref state)) < 128) { result |= tmp << 14; } else { result |= (tmp & 0x7f) << 14; if ((tmp = ReadRawByte(ref buffer, ref state)) < 128) { result |= tmp << 21; } else { result |= (tmp & 0x7f) << 21; result |= (tmp = ReadRawByte(ref buffer, ref state)) << 28; if (tmp >= 128) { // Discard upper 32 bits. for (int i = 0; i < 5; i++) { if (ReadRawByte(ref buffer, ref state) < 128) { return((uint)result); } } throw InvalidProtocolBufferException.MalformedVarint(); } } } } return((uint)result); }
/// <summary> /// Reads a string assuming that it is spread across multiple spans in a <see cref="ReadOnlySequence{T}"/>. /// </summary> private static string ReadStringSlow(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state, int length) { ValidateCurrentLimit(ref buffer, ref state, length); #if GOOGLE_PROTOBUF_SUPPORT_FAST_STRING if (IsDataAvailable(ref state, length)) { // Read string data into a temporary buffer, either stackalloc'ed or from ArrayPool // Once all data is read then call Encoding.GetString on buffer and return to pool if needed. byte[] byteArray = null; Span <byte> byteSpan = length <= StackallocThreshold ? stackalloc byte[length] : (byteArray = ArrayPool <byte> .Shared.Rent(length)); try { unsafe { fixed(byte *pByteSpan = &MemoryMarshal.GetReference(byteSpan)) { // Compiler doesn't like that a potentially stackalloc'd Span<byte> is being used // in a method with a "ref Span<byte> buffer" argument. If the stackalloc'd span was assigned // to the ref argument then bad things would happen. We'll never do that so it is ok. // Make compiler happy by passing a new span created from pointer. var tempSpan = new Span <byte>(pByteSpan, byteSpan.Length); ReadRawBytesIntoSpan(ref buffer, ref state, length, tempSpan); return(WritingPrimitives.Utf8Encoding.GetString(pByteSpan, length)); } } } finally { if (byteArray != null) { ArrayPool <byte> .Shared.Return(byteArray); } } } #endif // Slow path: Build a byte array first then copy it. // This will be called when reading from a Stream because we don't know the length of the stream, // or there is not enough data in the sequence. If there is not enough data then ReadRawBytes will // throw an exception. return(WritingPrimitives.Utf8Encoding.GetString(ReadRawBytes(ref buffer, ref state, length), 0, length)); }
/// <summary> /// Parses a 32-bit little-endian integer. /// </summary> public static uint ParseRawLittleEndian32(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state) { const int uintLength = sizeof(uint); const int ulongLength = sizeof(ulong); if (state.bufferPos + ulongLength > state.bufferSize) { return(ParseRawLittleEndian32SlowPath(ref buffer, ref state)); } // ReadUInt32LittleEndian is many times slower than ReadUInt64LittleEndian (at least on some runtimes) // so it's faster better to use ReadUInt64LittleEndian and truncate the result. uint result = (uint)BinaryPrimitives.ReadUInt64LittleEndian(buffer.Slice(state.bufferPos, ulongLength)); state.bufferPos += uintLength; return(result); }