/// <summary> /// Same code as ReadRawVarint32, but read each byte individually, checking for /// buffer overflow. /// </summary> private uint SlowReadRawVarint32() { int tmp = ReadRawByte(); if (tmp < 128) { return((uint)tmp); } int result = tmp & 0x7f; if ((tmp = ReadRawByte()) < 128) { result |= tmp << 7; } else { result |= (tmp & 0x7f) << 7; if ((tmp = ReadRawByte()) < 128) { result |= tmp << 14; } else { result |= (tmp & 0x7f) << 14; if ((tmp = ReadRawByte()) < 128) { result |= tmp << 21; } else { result |= (tmp & 0x7f) << 21; result |= (tmp = ReadRawByte()) << 28; if (tmp >= 128) { // Discard upper 32 bits. for (int i = 0; i < 5; i++) { if (ReadRawByte() < 128) { return((uint)result); } } throw InvalidProtocolBufferException.MalformedVarint(); } } } } return((uint)result); }
private static uint ParseRawVarint32SlowPath(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state) { int tmp = ReadRawByte(ref buffer, ref state); if (tmp < 128) { return((uint)tmp); } int result = tmp & 0x7f; if ((tmp = ReadRawByte(ref buffer, ref state)) < 128) { result |= tmp << 7; } else { result |= (tmp & 0x7f) << 7; if ((tmp = ReadRawByte(ref buffer, ref state)) < 128) { result |= tmp << 14; } else { result |= (tmp & 0x7f) << 14; if ((tmp = ReadRawByte(ref buffer, ref state)) < 128) { result |= tmp << 21; } else { result |= (tmp & 0x7f) << 21; result |= (tmp = ReadRawByte(ref buffer, ref state)) << 28; if (tmp >= 128) { // Discard upper 32 bits. for (int i = 0; i < 5; i++) { if (ReadRawByte(ref buffer, ref state) < 128) { return((uint)result); } } throw InvalidProtocolBufferException.MalformedVarint(); } } } } return((uint)result); }
/// <summary> /// Parses the given bytes using ReadRawVarint32() and ReadRawVarint64() and /// expects them to fail with an InvalidProtocolBufferException whose /// description matches the given one. /// </summary> private static void AssertReadVarintFailure(InvalidProtocolBufferException expected, byte[] data) { CodedInputStream input = new CodedInputStream(data); var exception = Assert.Throws <InvalidProtocolBufferException>(() => input.ReadRawVarint32()); Assert.AreEqual(expected.Message, exception.Message); input = new CodedInputStream(data); exception = Assert.Throws <InvalidProtocolBufferException>(() => input.ReadRawVarint64()); Assert.AreEqual(expected.Message, exception.Message); // Make sure we get the same error when reading directly from a Stream. exception = Assert.Throws <InvalidProtocolBufferException>(() => CodedInputStream.ReadRawVarint32(new MemoryStream(data))); Assert.AreEqual(expected.Message, exception.Message); }
private static void MergeFieldMask(IMessage message, JsonToken token) { if (token.Type != JsonToken.TokenType.StringValue) { throw InvalidProtocolBufferException.OnThrowMessage("Expected string value for FieldMask"); } // TODO: Do we *want* to remove empty entries? Probably okay to treat "" as "no paths", but "foo,,bar"? string[] jsonPaths = token.StringValue.Split(FieldMaskPathSeparators, StringSplitOptions.RemoveEmptyEntries); IList messagePaths = (IList)message.Descriptor.Fields[FieldMask.PathsFieldNumber].Accessor.GetValue(message); foreach (var path in jsonPaths) { messagePaths.Add(ToSnakeCase(path)); } }
// Ported from src/google/protobuf/util/internal/utility.cc private static string ToSnakeCase(string text) { var builder = new StringBuilder(text.Length * 2); // Note: this is probably unnecessary now, but currently retained to be as close as possible to the // C++, whilst still throwing an exception on underscores. bool wasNotUnderscore = false; // Initialize to false for case 1 (below) bool wasNotCap = false; for (int i = 0; i < text.Length; i++) { char c = text[i]; if (c >= 'A' && c <= 'Z') // ascii_isupper { // Consider when the current character B is capitalized: // 1) At beginning of input: "B..." => "b..." // (e.g. "Biscuit" => "biscuit") // 2) Following a lowercase: "...aB..." => "...a_b..." // (e.g. "gBike" => "g_bike") // 3) At the end of input: "...AB" => "...ab" // (e.g. "GoogleLAB" => "google_lab") // 4) Followed by a lowercase: "...ABc..." => "...a_bc..." // (e.g. "GBike" => "g_bike") if (wasNotUnderscore && // case 1 out (wasNotCap || // case 2 in, case 3 out (i + 1 < text.Length && // case 3 out (text[i + 1] >= 'a' && text[i + 1] <= 'z')))) // ascii_islower(text[i + 1]) { // case 4 in // We add an underscore for case 2 and case 4. builder.Append('_'); } // ascii_tolower, but we already know that c *is* an upper case ASCII character... builder.Append((char)(c + 'a' - 'A')); wasNotUnderscore = true; wasNotCap = false; } else { builder.Append(c); if (c == '_') { throw InvalidProtocolBufferException.OnThrowMessage($"Invalid field mask: {text}"); } wasNotUnderscore = true; wasNotCap = true; } } return(builder.ToString()); }
public static void ReadGroup(ref ParseContext ctx, IMessage message) { if (ctx.state.recursionDepth >= ctx.state.recursionLimit) { throw InvalidProtocolBufferException.RecursionLimitExceeded(); } ++ctx.state.recursionDepth; uint tag = ctx.state.lastTag; int fieldNumber = WireFormat.GetTagFieldNumber(tag); ReadRawMessage(ref ctx, message); CheckLastTagWas(ref ctx.state, WireFormat.MakeTag(fieldNumber, WireFormat.WireType.EndGroup)); --ctx.state.recursionDepth; }
/// <summary> /// Reads an embedded message field value from the stream. /// </summary> public void ReadMessage(IMessage builder) { int length = ReadLength(); if (recursionDepth >= recursionLimit) { throw InvalidProtocolBufferException.RecursionLimitExceeded(); } int oldLimit = PushLimit(length); ++recursionDepth; builder.MergeFrom(this); CheckLastTagWas(0); --recursionDepth; PopLimit(oldLimit); }
public static string ReadRawString(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state, int length) { // No need to read any data for an empty string. if (length == 0) { return(string.Empty); } if (length < 0) { throw InvalidProtocolBufferException.NegativeSize(); } #if GOOGLE_PROTOBUF_SUPPORT_FAST_STRING if (length <= state.bufferSize - state.bufferPos && length > 0) { // Fast path: all bytes to decode appear in the same span. ReadOnlySpan <byte> data = buffer.Slice(state.bufferPos, length); string value; unsafe { fixed(byte *sourceBytes = &MemoryMarshal.GetReference(data)) { value = CodedOutputStream.Utf8Encoding.GetString(sourceBytes, length); } } state.bufferPos += length; return(value); } #endif var decoder = CodedOutputStream.Utf8Encoding.GetDecoder(); // TODO: even if GOOGLE_PROTOBUF_SUPPORT_FAST_STRING is not supported, // we could still create a string efficiently by using Utf8Encoding.GetString(byte[] bytes, int index, int count) // whenever the buffer is backed by a byte array (and avoid creating a new byte array), but the problem is // there is no way to get the underlying byte array from a span. // TODO: in case the string spans multiple buffer segments, creating a char[] and decoding into it and then // creating a string from that array might be more efficient than creating a string from the copied bytes. // Slow path: Build a byte array first then copy it. return(CodedOutputStream.Utf8Encoding.GetString(ReadRawBytes(ref buffer, ref state, length), 0, length)); }
/// <summary> /// Reads a raw varint from the stream. /// </summary> internal ulong ReadRawVarint64() { int shift = 0; ulong result = 0; while (shift < 64) { byte b = ReadRawByte(); result |= (ulong)(b & 0x7F) << shift; if ((b & 0x80) == 0) { return(result); } shift += 7; } throw InvalidProtocolBufferException.MalformedVarint(); }
// Well-known types end up in a property called "value" in the JSON. As there's no longer a @type property // in the given JSON token stream, we should *only* have tokens of start-object, name("value"), the value // itself, and then end-object. private void MergeWellKnownTypeAnyBody(IMessage body, JsonTokenizer tokenizer) { var token = tokenizer.Next(); // Definitely start-object; checked in previous method token = tokenizer.Next(); // TODO: What about an absent Int32Value, for example? if (token.Type != JsonToken.TokenType.Name || token.StringValue != JsonFormatter.AnyWellKnownTypeValueField) { throw InvalidProtocolBufferException.OnThrowMessage($"Expected '{JsonFormatter.AnyWellKnownTypeValueField}' property for well-known type Any body"); } Merge(body, tokenizer); token = tokenizer.Next(); if (token.Type != JsonToken.TokenType.EndObject) { throw InvalidProtocolBufferException.OnThrowMessage($"Expected end-object token after @type/value for well-known type"); } }
private static void MergeDuration(IMessage message, JsonToken token) { if (token.Type != JsonToken.TokenType.StringValue) { throw InvalidProtocolBufferException.OnThrowMessage("Expected string value for Duration"); } var match = DurationRegex.Match(token.StringValue); if (!match.Success) { throw InvalidProtocolBufferException.OnThrowMessage("Invalid Duration value: " + token.StringValue); } var sign = match.Groups["sign"].Value; var secondsText = match.Groups["int"].Value; // Prohibit leading insignficant zeroes if (secondsText[0] == '0' && secondsText.Length > 1) { throw InvalidProtocolBufferException.OnThrowMessage("Invalid Duration value: " + token.StringValue); } var subseconds = match.Groups["subseconds"].Value; var multiplier = sign == "-" ? -1 : 1; try { long seconds = long.Parse(secondsText, CultureInfo.InvariantCulture) * multiplier; int nanos = 0; if (subseconds != "") { // This should always work, as we've got 1-9 digits. int parsedFraction = int.Parse(subseconds.Substring(1)); nanos = parsedFraction * SubsecondScalingFactors[subseconds.Length] * multiplier; } if (!Duration.IsNormalized(seconds, nanos)) { throw InvalidProtocolBufferException.OnThrowMessage($"Invalid Duration value: {token.StringValue}"); } message.Descriptor.Fields[Duration.SecondsFieldNumber].Accessor.SetValue(message, seconds); message.Descriptor.Fields[Duration.NanosFieldNumber].Accessor.SetValue(message, nanos); } catch (FormatException) { throw InvalidProtocolBufferException.OnThrowMessage($"Invalid Duration value: {token.StringValue}"); } }
private static ulong ParseRawVarint64SlowPath(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state) { int shift = 0; ulong result = 0; do { byte b = ReadRawByte(ref buffer, ref state); result |= (ulong)(b & 0x7F) << shift; if (b < 0x80) { return(result); } shift += 7; }while (shift < 64); throw InvalidProtocolBufferException.MalformedVarint(); }
/// <summary> /// Reads a fixed size of bytes from the input. /// </summary> /// <exception cref="InvalidProtocolBufferException"> /// the end of the stream or the current limit was reached /// </exception> public static byte[] ReadRawBytes(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state, int size) { if (size < 0) { throw InvalidProtocolBufferException.NegativeSize(); } if (size <= state.bufferSize - state.bufferPos) { // We have all the bytes we need already. byte[] bytes = new byte[size]; buffer.Slice(state.bufferPos, size).CopyTo(bytes); state.bufferPos += size; return(bytes); } return(ReadRawBytesSlow(ref buffer, ref state, size)); }
/// <summary> /// Reads and discards <paramref name="size"/> bytes. /// </summary> /// <exception cref="InvalidProtocolBufferException">the end of the stream /// or the current limit was reached</exception> private void SkipRawBytes(int size) { if (size < 0) { throw InvalidProtocolBufferException.NegativeSize(); } if (totalBytesRetired + bufferPos + size > currentLimit) { // Read to the end of the stream anyway. SkipRawBytes(currentLimit - totalBytesRetired - bufferPos); // Then fail. throw InvalidProtocolBufferException.TruncatedMessage(); } if (size <= bufferSize - bufferPos) { // We have all the bytes we need already. bufferPos += size; } else { // Skipping more bytes than are in the buffer. First skip what we have. int pos = bufferSize - bufferPos; // ROK 5/7/2013 Issue #54: should retire all bytes in buffer (bufferSize) // totalBytesRetired += pos; totalBytesRetired += bufferSize; bufferPos = 0; bufferSize = 0; // Then skip directly from the InputStream for the rest. if (pos < size) { if (input == null) { throw InvalidProtocolBufferException.TruncatedMessage(); } SkipImpl(size - pos); totalBytesRetired += size - pos; } } }
/// <summary> /// Sets currentLimit to (current position) + byteLimit. This is called /// when descending into a length-delimited embedded message. The previous /// limit is returned. /// </summary> /// <returns>The old limit.</returns> public static int PushLimit(ref ParserInternalState state, int byteLimit) { if (byteLimit < 0) { throw InvalidProtocolBufferException.NegativeSize(); } byteLimit += state.totalBytesRetired + state.bufferPos; int oldLimit = state.currentLimit; if (byteLimit > oldLimit) { throw InvalidProtocolBufferException.TruncatedMessage(); } state.currentLimit = byteLimit; RecomputeBufferSizeAfterLimit(ref state); return(oldLimit); }
public void ReadMapItor(Action <CodedInputStream> decoder) { int length = ReadLength(); if (length == 0) { return; } if (recursionDepth >= recursionLimit) { throw InvalidProtocolBufferException.RecursionLimitExceeded(); } int oldLimit = PushLimit(length); ++recursionDepth; decoder.Invoke(this); --recursionDepth; PopLimit(oldLimit); }
/// <summary> /// Sets currentLimit to (current position) + byteLimit. This is called /// when descending into a length-delimited embedded message. The previous /// limit is returned. /// </summary> /// <returns>The old limit.</returns> internal int PushLimit(int byteLimit) { if (byteLimit < 0) { throw InvalidProtocolBufferException.NegativeSize(); } byteLimit += totalBytesRetired + bufferPos; int oldLimit = currentLimit; if (byteLimit > oldLimit) { throw InvalidProtocolBufferException.TruncatedMessage(); } currentLimit = byteLimit; RecomputeBufferSizeAfterLimit(); return(oldLimit); }
/// <summary> /// Parses the given bytes using ReadRawVarint32() and ReadRawVarint64() and /// expects them to fail with an InvalidProtocolBufferException whose /// description matches the given one. /// </summary> private static void AssertReadVarintFailure(InvalidProtocolBufferException expected, byte[] data) { CodedInputStream input = new CodedInputStream(data); var exception = Assert.Throws <InvalidProtocolBufferException>(() => input.ReadRawVarint32()); Assert.AreEqual(expected.Message, exception.Message); input = new CodedInputStream(data); exception = Assert.Throws <InvalidProtocolBufferException>(() => input.ReadRawVarint64()); Assert.AreEqual(expected.Message, exception.Message); AssertReadFromParseContext(new ReadOnlySequence <byte>(data), (ref ParseContext ctx) => { try { ctx.ReadUInt32(); Assert.Fail(); } catch (InvalidProtocolBufferException ex) { Assert.AreEqual(expected.Message, ex.Message); } }, false); AssertReadFromParseContext(new ReadOnlySequence <byte>(data), (ref ParseContext ctx) => { try { ctx.ReadUInt64(); Assert.Fail(); } catch (InvalidProtocolBufferException ex) { Assert.AreEqual(expected.Message, ex.Message); } }, false); // Make sure we get the same error when reading directly from a Stream. exception = Assert.Throws <InvalidProtocolBufferException>(() => CodedInputStream.ReadRawVarint32(new MemoryStream(data))); Assert.AreEqual(expected.Message, exception.Message); }
/// <summary> /// Reads and discards <paramref name="size"/> bytes. /// </summary> /// <exception cref="InvalidProtocolBufferException">the end of the stream /// or the current limit was reached</exception> public static void SkipRawBytes(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state, int size) { if (size < 0) { throw InvalidProtocolBufferException.NegativeSize(); } if (state.totalBytesRetired + state.bufferPos + size > state.currentLimit) { // Read to the end of the stream anyway. SkipRawBytes(ref buffer, ref state, state.currentLimit - state.totalBytesRetired - state.bufferPos); // Then fail. throw InvalidProtocolBufferException.TruncatedMessage(); } if (size <= state.bufferSize - state.bufferPos) { // We have all the bytes we need already. state.bufferPos += size; } else { // Skipping more bytes than are in the buffer. First skip what we have. int pos = state.bufferSize - state.bufferPos; state.bufferPos = state.bufferSize; // TODO: If our segmented buffer is backed by a Stream that is seekable, we could skip the bytes more efficiently // by simply updating stream's Position property. This used to be supported in the past, but the support was dropped // because it would make the segmentedBufferHelper more complex. Support can be reintroduced if needed. state.segmentedBufferHelper.RefillBuffer(ref buffer, ref state, true); while (size - pos > state.bufferSize) { pos += state.bufferSize; state.bufferPos = state.bufferSize; state.segmentedBufferHelper.RefillBuffer(ref buffer, ref state, true); } state.bufferPos = size - pos; } }
/// <summary> /// Reads an embedded message field value from the stream. /// </summary> public void ReadMessage(IMessage builder) { int length = ReadLength(); if (recursionDepth >= recursionLimit) { throw InvalidProtocolBufferException.RecursionLimitExceeded(); } int oldLimit = PushLimit(length); ++recursionDepth; builder.MergeFrom(this); CheckReadEndOfStreamTag(); // Check that we've read exactly as much data as expected. if (!ReachedLimit) { throw InvalidProtocolBufferException.TruncatedMessage(); } --recursionDepth; PopLimit(oldLimit); }
private static object ParseMapKey(FieldDescriptor field, string keyText) { switch (field.FieldType) { case FieldType.Bool: if (keyText == "true") { return(true); } if (keyText == "false") { return(false); } throw InvalidProtocolBufferException.OnThrowMessage("Invalid string for bool map key: " + keyText); case FieldType.String: return(keyText); case FieldType.Int32: case FieldType.SInt32: case FieldType.SFixed32: return(ParseNumericString(keyText, int.Parse)); case FieldType.UInt32: case FieldType.Fixed32: return(ParseNumericString(keyText, uint.Parse)); case FieldType.Int64: case FieldType.SInt64: case FieldType.SFixed64: return(ParseNumericString(keyText, long.Parse)); case FieldType.UInt64: case FieldType.Fixed64: return(ParseNumericString(keyText, ulong.Parse)); default: throw InvalidProtocolBufferException.OnThrowMessage("Invalid field type for map: " + field.FieldType); } }
public static void ReadMessage(ref ParseContext ctx, IMessage message) { int length = ParsingPrimitives.ParseLength(ref ctx.buffer, ref ctx.state); if (ctx.state.recursionDepth >= ctx.state.recursionLimit) { throw InvalidProtocolBufferException.RecursionLimitExceeded(); } int oldLimit = SegmentedBufferHelper.PushLimit(ref ctx.state, length); ++ctx.state.recursionDepth; ReadRawMessage(ref ctx, message); CheckReadEndOfStreamTag(ref ctx.state); // Check that we've read exactly as much data as expected. if (!SegmentedBufferHelper.IsReachedLimit(ref ctx.state)) { throw InvalidProtocolBufferException.TruncatedMessage(); } --ctx.state.recursionDepth; SegmentedBufferHelper.PopLimit(ref ctx.state, oldLimit); }
private void SkipGroup() { // Note: Currently we expect this to be the way that groups are read. We could put the recursion // depth changes into the ReadTag method instead, potentially... recursionDepth++; if (recursionDepth >= recursionLimit) { throw InvalidProtocolBufferException.RecursionLimitExceeded(); } uint tag; do { tag = ReadTag(); if (tag == 0) { throw InvalidProtocolBufferException.TruncatedMessage(); } // This recursion will allow us to handle nested groups. SkipLastField(); } while (WireFormat.GetTagWireType(tag) != WireFormat.WireType.EndGroup); recursionDepth--; }
/// <summary> /// Skip a group. /// </summary> internal void SkipGroup(uint startGroupTag) { // Note: Currently we expect this to be the way that groups are read. We could put the recursion // depth changes into the ReadTag method instead, potentially... recursionDepth++; if (recursionDepth >= recursionLimit) { throw InvalidProtocolBufferException.RecursionLimitExceeded(); } uint tag; while (true) { tag = ReadTag(); if (tag == 0) { throw InvalidProtocolBufferException.TruncatedMessage(); } // Can't call SkipLastField for this case- that would throw. if (WireFormat.GetTagWireType(tag) == WireFormat.WireType.EndGroup) { break; } // This recursion will allow us to handle nested groups. SkipLastField(); } int startField = WireFormat.GetTagFieldNumber(startGroupTag); int endField = WireFormat.GetTagFieldNumber(tag); if (startField != endField) { throw new InvalidProtocolBufferException( string.Format("Mismatched end-group tag. Started with field {0}; ended with field {1}", startField, endField)); } recursionDepth--; }
/// <summary> /// Skip a group. /// </summary> public static void SkipGroup(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state, uint startGroupTag) { // Note: Currently we expect this to be the way that groups are read. We could put the recursion // depth changes into the ReadTag method instead, potentially... state.recursionDepth++; if (state.recursionDepth >= state.recursionLimit) { throw InvalidProtocolBufferException.RecursionLimitExceeded(); } uint tag; while (true) { tag = ParsingPrimitives.ParseTag(ref buffer, ref state); if (tag == 0) { throw InvalidProtocolBufferException.TruncatedMessage(); } // Can't call SkipLastField for this case- that would throw. if (WireFormat.GetTagWireType(tag) == WireFormat.WireType.EndGroup) { break; } // This recursion will allow us to handle nested groups. SkipLastField(ref buffer, ref state); } int startField = WireFormat.GetTagFieldNumber(startGroupTag); int endField = WireFormat.GetTagFieldNumber(tag); if (startField != endField) { throw new InvalidProtocolBufferException( $"Mismatched end-group tag. Started with field {startField}; ended with field {endField}"); } state.recursionDepth--; }
public void ReadVarint() { AssertReadVarint(Bytes(0x00), 0); AssertReadVarint(Bytes(0x01), 1); AssertReadVarint(Bytes(0x7f), 127); // 14882 AssertReadVarint(Bytes(0xa2, 0x74), (0x22 << 0) | (0x74 << 7)); // 2961488830 AssertReadVarint(Bytes(0xbe, 0xf7, 0x92, 0x84, 0x0b), (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) | (0x0bL << 28)); // 64-bit // 7256456126 AssertReadVarint(Bytes(0xbe, 0xf7, 0x92, 0x84, 0x1b), (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) | (0x1bL << 28)); // 41256202580718336 AssertReadVarint(Bytes(0x80, 0xe6, 0xeb, 0x9c, 0xc3, 0xc9, 0xa4, 0x49), (0x00 << 0) | (0x66 << 7) | (0x6b << 14) | (0x1c << 21) | (0x43L << 28) | (0x49L << 35) | (0x24L << 42) | (0x49L << 49)); // 11964378330978735131 AssertReadVarint(Bytes(0x9b, 0xa8, 0xf9, 0xc2, 0xbb, 0xd6, 0x80, 0x85, 0xa6, 0x01), (0x1b << 0) | (0x28 << 7) | (0x79 << 14) | (0x42 << 21) | (0x3bUL << 28) | (0x56UL << 35) | (0x00UL << 42) | (0x05UL << 49) | (0x26UL << 56) | (0x01UL << 63)); // Failures AssertReadVarintFailure( InvalidProtocolBufferException.MalformedVarint(), Bytes(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00)); AssertReadVarintFailure( InvalidProtocolBufferException.TruncatedMessage(), Bytes(0x80)); }
private void MergeMapField(IMessage message, FieldDescriptor field, JsonTokenizer tokenizer) { // Map fields are always objects, even if the values are well-known types: ParseSingleValue handles those. var token = tokenizer.Next(); if (token.Type != JsonToken.TokenType.StartObject) { throw InvalidProtocolBufferException.OnThrowMessage("Expected an object to populate a map"); } var type = field.MessageType; var keyField = type.FindFieldByNumber(1); var valueField = type.FindFieldByNumber(2); if (keyField == null || valueField == null) { throw InvalidProtocolBufferException.OnThrowMessage("Invalid map field: " + field.FullName); } IDictionary dictionary = (IDictionary)field.Accessor.GetValue(message); while (true) { token = tokenizer.Next(); if (token.Type == JsonToken.TokenType.EndObject) { return; } object key = ParseMapKey(keyField, token.StringValue); object value = ParseSingleValue(valueField, tokenizer); if (value == null) { throw InvalidProtocolBufferException.OnThrowMessage("Map values must not be null"); } dictionary[key] = value; } }
/// <summary> /// Reads a fixed size of bytes from the input. /// </summary> /// <exception cref="InvalidProtocolBufferException"> /// the end of the stream or the current limit was reached /// </exception> internal byte[] ReadRawBytes(int size) { if (size < 0) { throw InvalidProtocolBufferException.NegativeSize(); } if (totalBytesRetired + bufferPos + size > currentLimit) { // Read to the end of the stream (up to the current limit) anyway. SkipRawBytes(currentLimit - totalBytesRetired - bufferPos); // Then fail. throw InvalidProtocolBufferException.TruncatedMessage(); } if (size <= bufferSize - bufferPos) { // We have all the bytes we need already. byte[] bytes = new byte[size]; ByteArray.Copy(buffer, bufferPos, bytes, 0, size); bufferPos += size; return(bytes); } else if (size < buffer.Length) { // Reading more bytes than are in the buffer, but not an excessive number // of bytes. We can safely allocate the resulting array ahead of time. // First copy what we have. byte[] bytes = new byte[size]; int pos = bufferSize - bufferPos; ByteArray.Copy(buffer, bufferPos, bytes, 0, pos); bufferPos = bufferSize; // We want to use RefillBuffer() and then copy from the buffer into our // byte array rather than reading directly into our byte array because // the input may be unbuffered. RefillBuffer(true); while (size - pos > bufferSize) { Buffer.BlockCopy(buffer, 0, bytes, pos, bufferSize); pos += bufferSize; bufferPos = bufferSize; RefillBuffer(true); } ByteArray.Copy(buffer, 0, bytes, pos, size - pos); bufferPos = size - pos; return(bytes); } else { // The size is very large. For security reasons, we can't allocate the // entire byte array yet. The size comes directly from the input, so a // maliciously-crafted message could provide a bogus very large size in // order to trick the app into allocating a lot of memory. We avoid this // by allocating and reading only a small chunk at a time, so that the // malicious message must actually *be* extremely large to cause // problems. Meanwhile, we limit the allowed size of a message elsewhere. // Remember the buffer markers since we'll have to copy the bytes out of // it later. int originalBufferPos = bufferPos; int originalBufferSize = bufferSize; // Mark the current buffer consumed. totalBytesRetired += bufferSize; bufferPos = 0; bufferSize = 0; // Read all the rest of the bytes we need. int sizeLeft = size - (originalBufferSize - originalBufferPos); List <byte[]> chunks = new List <byte[]>(); while (sizeLeft > 0) { byte[] chunk = new byte[Math.Min(sizeLeft, buffer.Length)]; int pos = 0; while (pos < chunk.Length) { int n = (input == null) ? -1 : input.Read(chunk, pos, chunk.Length - pos); if (n <= 0) { throw InvalidProtocolBufferException.TruncatedMessage(); } totalBytesRetired += n; pos += n; } sizeLeft -= chunk.Length; chunks.Add(chunk); } // OK, got everything. Now concatenate it all into one buffer. byte[] bytes = new byte[size]; // Start by copying the leftover bytes from this.buffer. int newPos = originalBufferSize - originalBufferPos; ByteArray.Copy(buffer, originalBufferPos, bytes, 0, newPos); // And now all the chunks. foreach (byte[] chunk in chunks) { Buffer.BlockCopy(chunk, 0, bytes, newPos, chunk.Length); newPos += chunk.Length; } // Done. return(bytes); } }
/// <summary> /// Merges the given message using data from the given tokenizer. In most cases, the next /// token should be a "start object" token, but wrapper types and nullity can invalidate /// that assumption. This is implemented as an LL(1) recursive descent parser over the stream /// of tokens provided by the tokenizer. This token stream is assumed to be valid JSON, with the /// tokenizer performing that validation - but not every token stream is valid "protobuf JSON". /// </summary> private void Merge(IMessage message, JsonTokenizer tokenizer) { if (tokenizer.ObjectDepth > settings.RecursionLimit) { throw InvalidProtocolBufferException.JsonRecursionLimitExceeded(); } if (message.Descriptor.IsWellKnownType) { Action <JsonParser, IMessage, JsonTokenizer> handler; if (WellKnownTypeHandlers.TryGetValue(message.Descriptor.FullName, out handler)) { handler(this, message, tokenizer); return; } // Well-known types with no special handling continue in the normal way. } var token = tokenizer.Next(); if (token.Type != JsonToken.TokenType.StartObject) { throw new InvalidProtocolBufferException("Expected an object"); } var descriptor = message.Descriptor; var jsonFieldMap = descriptor.Fields.ByJsonName(); // All the oneof fields we've already accounted for - we can only see each of them once. // The set is created lazily to avoid the overhead of creating a set for every message // we parsed, when oneofs are relatively rare. HashSet <OneofDescriptor> seenOneofs = null; while (true) { token = tokenizer.Next(); if (token.Type == JsonToken.TokenType.EndObject) { return; } if (token.Type != JsonToken.TokenType.Name) { throw new InvalidOperationException("Unexpected token type " + token.Type); } string name = token.StringValue; FieldDescriptor field; if (jsonFieldMap.TryGetValue(name, out field)) { if (field.ContainingOneof != null) { if (seenOneofs == null) { seenOneofs = new HashSet <OneofDescriptor>(); } if (!seenOneofs.Add(field.ContainingOneof)) { throw new InvalidProtocolBufferException($"Multiple values specified for oneof {field.ContainingOneof.Name}"); } } MergeField(message, field, tokenizer); } else { // TODO: Is this what we want to do? If not, we'll need to skip the value, // which may be an object or array. (We might want to put code in the tokenizer // to do that.) throw new InvalidProtocolBufferException("Unknown field: " + name); } } }
/// <summary> /// Reads a fixed size of bytes from the input. /// </summary> /// <exception cref="InvalidProtocolBufferException"> /// the end of the stream or the current limit was reached /// </exception> public static byte[] ReadRawBytes(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state, int size) { if (size < 0) { throw InvalidProtocolBufferException.NegativeSize(); } if (state.totalBytesRetired + state.bufferPos + size > state.currentLimit) { // Read to the end of the stream (up to the current limit) anyway. SkipRawBytes(ref buffer, ref state, state.currentLimit - state.totalBytesRetired - state.bufferPos); // Then fail. throw InvalidProtocolBufferException.TruncatedMessage(); } if (size <= state.bufferSize - state.bufferPos) { // We have all the bytes we need already. byte[] bytes = new byte[size]; buffer.Slice(state.bufferPos, size).CopyTo(bytes); state.bufferPos += size; return(bytes); } else if (size < buffer.Length || size < state.segmentedBufferHelper.TotalLength) { // Reading more bytes than are in the buffer, but not an excessive number // of bytes. We can safely allocate the resulting array ahead of time. // First copy what we have. byte[] bytes = new byte[size]; var bytesSpan = new Span <byte>(bytes); int pos = state.bufferSize - state.bufferPos; buffer.Slice(state.bufferPos, pos).CopyTo(bytesSpan.Slice(0, pos)); state.bufferPos = state.bufferSize; // We want to use RefillBuffer() and then copy from the buffer into our // byte array rather than reading directly into our byte array because // the input may be unbuffered. state.segmentedBufferHelper.RefillBuffer(ref buffer, ref state, true); while (size - pos > state.bufferSize) { buffer.Slice(0, state.bufferSize) .CopyTo(bytesSpan.Slice(pos, state.bufferSize)); pos += state.bufferSize; state.bufferPos = state.bufferSize; state.segmentedBufferHelper.RefillBuffer(ref buffer, ref state, true); } buffer.Slice(0, size - pos) .CopyTo(bytesSpan.Slice(pos, size - pos)); state.bufferPos = size - pos; return(bytes); } else { // The size is very large. For security reasons, we can't allocate the // entire byte array yet. The size comes directly from the input, so a // maliciously-crafted message could provide a bogus very large size in // order to trick the app into allocating a lot of memory. We avoid this // by allocating and reading only a small chunk at a time, so that the // malicious message must actually *be* extremely large to cause // problems. Meanwhile, we limit the allowed size of a message elsewhere. List <byte[]> chunks = new List <byte[]>(); int pos = state.bufferSize - state.bufferPos; byte[] firstChunk = new byte[pos]; buffer.Slice(state.bufferPos, pos).CopyTo(firstChunk); chunks.Add(firstChunk); state.bufferPos = state.bufferSize; // Read all the rest of the bytes we need. int sizeLeft = size - pos; while (sizeLeft > 0) { state.segmentedBufferHelper.RefillBuffer(ref buffer, ref state, true); byte[] chunk = new byte[Math.Min(sizeLeft, state.bufferSize)]; buffer.Slice(0, chunk.Length) .CopyTo(chunk); state.bufferPos += chunk.Length; sizeLeft -= chunk.Length; chunks.Add(chunk); } // OK, got everything. Now concatenate it all into one buffer. byte[] bytes = new byte[size]; int newPos = 0; foreach (byte[] chunk in chunks) { Buffer.BlockCopy(chunk, 0, bytes, newPos, chunk.Length); newPos += chunk.Length; } // Done. return(bytes); } }
/// <summary> /// Parses the given bytes using ReadRawVarint32() and ReadRawVarint64() and /// expects them to fail with an InvalidProtocolBufferException whose /// description matches the given one. /// </summary> private static void AssertReadVarintFailure(InvalidProtocolBufferException expected, byte[] data) { CodedInputStream input = new CodedInputStream(data); var exception = Assert.Throws<InvalidProtocolBufferException>(() => input.ReadRawVarint32()); Assert.AreEqual(expected.Message, exception.Message); input = new CodedInputStream(data); exception = Assert.Throws<InvalidProtocolBufferException>(() => input.ReadRawVarint64()); Assert.AreEqual(expected.Message, exception.Message); // Make sure we get the same error when reading directly from a Stream. exception = Assert.Throws<InvalidProtocolBufferException>(() => CodedInputStream.ReadRawVarint32(new MemoryStream(data))); Assert.AreEqual(expected.Message, exception.Message); }