/// <summary>
/// Same code as ReadRawVarint32, but read each byte individually, checking for
/// buffer overflow.
/// </summary>
private uint SlowReadRawVarint32()
{
int tmp = ReadRawByte();
if (tmp < 128)
{
return((uint)tmp);
}
int result = tmp & 0x7f;
if ((tmp = ReadRawByte()) < 128)
{
result |= tmp << 7;
}
else
{
result |= (tmp & 0x7f) << 7;
if ((tmp = ReadRawByte()) < 128)
{
result |= tmp << 14;
}
else
{
result |= (tmp & 0x7f) << 14;
if ((tmp = ReadRawByte()) < 128)
{
result |= tmp << 21;
}
else
{
result |= (tmp & 0x7f) << 21;
result |= (tmp = ReadRawByte()) << 28;
if (tmp >= 128)
{
// Discard upper 32 bits.
for (int i = 0; i < 5; i++)
{
if (ReadRawByte() < 128)
{
return((uint)result);
}
}
throw InvalidProtocolBufferException.MalformedVarint();
}
}
}
}
return((uint)result);
}
private static uint ParseRawVarint32SlowPath(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state)
{
int tmp = ReadRawByte(ref buffer, ref state);
if (tmp < 128)
{
return((uint)tmp);
}
int result = tmp & 0x7f;
if ((tmp = ReadRawByte(ref buffer, ref state)) < 128)
{
result |= tmp << 7;
}
else
{
result |= (tmp & 0x7f) << 7;
if ((tmp = ReadRawByte(ref buffer, ref state)) < 128)
{
result |= tmp << 14;
}
else
{
result |= (tmp & 0x7f) << 14;
if ((tmp = ReadRawByte(ref buffer, ref state)) < 128)
{
result |= tmp << 21;
}
else
{
result |= (tmp & 0x7f) << 21;
result |= (tmp = ReadRawByte(ref buffer, ref state)) << 28;
if (tmp >= 128)
{
// Discard upper 32 bits.
for (int i = 0; i < 5; i++)
{
if (ReadRawByte(ref buffer, ref state) < 128)
{
return((uint)result);
}
}
throw InvalidProtocolBufferException.MalformedVarint();
}
}
}
}
return((uint)result);
}
/// <summary>
/// Parses the given bytes using ReadRawVarint32() and ReadRawVarint64() and
/// expects them to fail with an InvalidProtocolBufferException whose
/// description matches the given one.
/// </summary>
private static void AssertReadVarintFailure(InvalidProtocolBufferException expected, byte[] data)
{
CodedInputStream input = new CodedInputStream(data);
var exception = Assert.Throws <InvalidProtocolBufferException>(() => input.ReadRawVarint32());
Assert.AreEqual(expected.Message, exception.Message);
input = new CodedInputStream(data);
exception = Assert.Throws <InvalidProtocolBufferException>(() => input.ReadRawVarint64());
Assert.AreEqual(expected.Message, exception.Message);
// Make sure we get the same error when reading directly from a Stream.
exception = Assert.Throws <InvalidProtocolBufferException>(() => CodedInputStream.ReadRawVarint32(new MemoryStream(data)));
Assert.AreEqual(expected.Message, exception.Message);
}
private static void MergeFieldMask(IMessage message, JsonToken token)
{
if (token.Type != JsonToken.TokenType.StringValue)
{
throw InvalidProtocolBufferException.OnThrowMessage("Expected string value for FieldMask");
}
// TODO: Do we *want* to remove empty entries? Probably okay to treat "" as "no paths", but "foo,,bar"?
string[] jsonPaths = token.StringValue.Split(FieldMaskPathSeparators, StringSplitOptions.RemoveEmptyEntries);
IList messagePaths = (IList)message.Descriptor.Fields[FieldMask.PathsFieldNumber].Accessor.GetValue(message);
foreach (var path in jsonPaths)
{
messagePaths.Add(ToSnakeCase(path));
}
}
// Ported from src/google/protobuf/util/internal/utility.cc
private static string ToSnakeCase(string text)
{
var builder = new StringBuilder(text.Length * 2);
// Note: this is probably unnecessary now, but currently retained to be as close as possible to the
// C++, whilst still throwing an exception on underscores.
bool wasNotUnderscore = false; // Initialize to false for case 1 (below)
bool wasNotCap = false;
for (int i = 0; i < text.Length; i++)
{
char c = text[i];
if (c >= 'A' && c <= 'Z') // ascii_isupper
{
// Consider when the current character B is capitalized:
// 1) At beginning of input: "B..." => "b..."
// (e.g. "Biscuit" => "biscuit")
// 2) Following a lowercase: "...aB..." => "...a_b..."
// (e.g. "gBike" => "g_bike")
// 3) At the end of input: "...AB" => "...ab"
// (e.g. "GoogleLAB" => "google_lab")
// 4) Followed by a lowercase: "...ABc..." => "...a_bc..."
// (e.g. "GBike" => "g_bike")
if (wasNotUnderscore && // case 1 out
(wasNotCap || // case 2 in, case 3 out
(i + 1 < text.Length && // case 3 out
(text[i + 1] >= 'a' && text[i + 1] <= 'z')))) // ascii_islower(text[i + 1])
{ // case 4 in
// We add an underscore for case 2 and case 4.
builder.Append('_');
}
// ascii_tolower, but we already know that c *is* an upper case ASCII character...
builder.Append((char)(c + 'a' - 'A'));
wasNotUnderscore = true;
wasNotCap = false;
}
else
{
builder.Append(c);
if (c == '_')
{
throw InvalidProtocolBufferException.OnThrowMessage($"Invalid field mask: {text}");
}
wasNotUnderscore = true;
wasNotCap = true;
}
}
return(builder.ToString());
}
public static void ReadGroup(ref ParseContext ctx, IMessage message)
{
if (ctx.state.recursionDepth >= ctx.state.recursionLimit)
{
throw InvalidProtocolBufferException.RecursionLimitExceeded();
}
++ctx.state.recursionDepth;
uint tag = ctx.state.lastTag;
int fieldNumber = WireFormat.GetTagFieldNumber(tag);
ReadRawMessage(ref ctx, message);
CheckLastTagWas(ref ctx.state, WireFormat.MakeTag(fieldNumber, WireFormat.WireType.EndGroup));
--ctx.state.recursionDepth;
}
/// <summary>
/// Reads an embedded message field value from the stream.
/// </summary>
public void ReadMessage(IMessage builder)
{
int length = ReadLength();
if (recursionDepth >= recursionLimit)
{
throw InvalidProtocolBufferException.RecursionLimitExceeded();
}
int oldLimit = PushLimit(length);
++recursionDepth;
builder.MergeFrom(this);
CheckLastTagWas(0);
--recursionDepth;
PopLimit(oldLimit);
}
public static string ReadRawString(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state, int length)
{
// No need to read any data for an empty string.
if (length == 0)
{
return(string.Empty);
}
if (length < 0)
{
throw InvalidProtocolBufferException.NegativeSize();
}
#if GOOGLE_PROTOBUF_SUPPORT_FAST_STRING
if (length <= state.bufferSize - state.bufferPos && length > 0)
{
// Fast path: all bytes to decode appear in the same span.
ReadOnlySpan <byte> data = buffer.Slice(state.bufferPos, length);
string value;
unsafe
{
fixed(byte *sourceBytes = &MemoryMarshal.GetReference(data))
{
value = CodedOutputStream.Utf8Encoding.GetString(sourceBytes, length);
}
}
state.bufferPos += length;
return(value);
}
#endif
var decoder = CodedOutputStream.Utf8Encoding.GetDecoder();
// TODO: even if GOOGLE_PROTOBUF_SUPPORT_FAST_STRING is not supported,
// we could still create a string efficiently by using Utf8Encoding.GetString(byte[] bytes, int index, int count)
// whenever the buffer is backed by a byte array (and avoid creating a new byte array), but the problem is
// there is no way to get the underlying byte array from a span.
// TODO: in case the string spans multiple buffer segments, creating a char[] and decoding into it and then
// creating a string from that array might be more efficient than creating a string from the copied bytes.
// Slow path: Build a byte array first then copy it.
return(CodedOutputStream.Utf8Encoding.GetString(ReadRawBytes(ref buffer, ref state, length), 0, length));
}
/// <summary>
/// Reads a raw varint from the stream.
/// </summary>
internal ulong ReadRawVarint64()
{
int shift = 0;
ulong result = 0;
while (shift < 64)
{
byte b = ReadRawByte();
result |= (ulong)(b & 0x7F) << shift;
if ((b & 0x80) == 0)
{
return(result);
}
shift += 7;
}
throw InvalidProtocolBufferException.MalformedVarint();
}
// Well-known types end up in a property called "value" in the JSON. As there's no longer a @type property
// in the given JSON token stream, we should *only* have tokens of start-object, name("value"), the value
// itself, and then end-object.
private void MergeWellKnownTypeAnyBody(IMessage body, JsonTokenizer tokenizer)
{
var token = tokenizer.Next(); // Definitely start-object; checked in previous method
token = tokenizer.Next();
// TODO: What about an absent Int32Value, for example?
if (token.Type != JsonToken.TokenType.Name || token.StringValue != JsonFormatter.AnyWellKnownTypeValueField)
{
throw InvalidProtocolBufferException.OnThrowMessage($"Expected '{JsonFormatter.AnyWellKnownTypeValueField}' property for well-known type Any body");
}
Merge(body, tokenizer);
token = tokenizer.Next();
if (token.Type != JsonToken.TokenType.EndObject)
{
throw InvalidProtocolBufferException.OnThrowMessage($"Expected end-object token after @type/value for well-known type");
}
}
private static void MergeDuration(IMessage message, JsonToken token)
{
if (token.Type != JsonToken.TokenType.StringValue)
{
throw InvalidProtocolBufferException.OnThrowMessage("Expected string value for Duration");
}
var match = DurationRegex.Match(token.StringValue);
if (!match.Success)
{
throw InvalidProtocolBufferException.OnThrowMessage("Invalid Duration value: " + token.StringValue);
}
var sign = match.Groups["sign"].Value;
var secondsText = match.Groups["int"].Value;
// Prohibit leading insignficant zeroes
if (secondsText[0] == '0' && secondsText.Length > 1)
{
throw InvalidProtocolBufferException.OnThrowMessage("Invalid Duration value: " + token.StringValue);
}
var subseconds = match.Groups["subseconds"].Value;
var multiplier = sign == "-" ? -1 : 1;
try
{
long seconds = long.Parse(secondsText, CultureInfo.InvariantCulture) * multiplier;
int nanos = 0;
if (subseconds != "")
{
// This should always work, as we've got 1-9 digits.
int parsedFraction = int.Parse(subseconds.Substring(1));
nanos = parsedFraction * SubsecondScalingFactors[subseconds.Length] * multiplier;
}
if (!Duration.IsNormalized(seconds, nanos))
{
throw InvalidProtocolBufferException.OnThrowMessage($"Invalid Duration value: {token.StringValue}");
}
message.Descriptor.Fields[Duration.SecondsFieldNumber].Accessor.SetValue(message, seconds);
message.Descriptor.Fields[Duration.NanosFieldNumber].Accessor.SetValue(message, nanos);
}
catch (FormatException)
{
throw InvalidProtocolBufferException.OnThrowMessage($"Invalid Duration value: {token.StringValue}");
}
}
private static ulong ParseRawVarint64SlowPath(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state)
{
int shift = 0;
ulong result = 0;
do
{
byte b = ReadRawByte(ref buffer, ref state);
result |= (ulong)(b & 0x7F) << shift;
if (b < 0x80)
{
return(result);
}
shift += 7;
}while (shift < 64);
throw InvalidProtocolBufferException.MalformedVarint();
}
/// <summary>
/// Reads a fixed size of bytes from the input.
/// </summary>
/// <exception cref="InvalidProtocolBufferException">
/// the end of the stream or the current limit was reached
/// </exception>
public static byte[] ReadRawBytes(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state, int size)
{
if (size < 0)
{
throw InvalidProtocolBufferException.NegativeSize();
}
if (size <= state.bufferSize - state.bufferPos)
{
// We have all the bytes we need already.
byte[] bytes = new byte[size];
buffer.Slice(state.bufferPos, size).CopyTo(bytes);
state.bufferPos += size;
return(bytes);
}
return(ReadRawBytesSlow(ref buffer, ref state, size));
}
/// <summary>
/// Reads and discards <paramref name="size"/> bytes.
/// </summary>
/// <exception cref="InvalidProtocolBufferException">the end of the stream
/// or the current limit was reached</exception>
private void SkipRawBytes(int size)
{
if (size < 0)
{
throw InvalidProtocolBufferException.NegativeSize();
}
if (totalBytesRetired + bufferPos + size > currentLimit)
{
// Read to the end of the stream anyway.
SkipRawBytes(currentLimit - totalBytesRetired - bufferPos);
// Then fail.
throw InvalidProtocolBufferException.TruncatedMessage();
}
if (size <= bufferSize - bufferPos)
{
// We have all the bytes we need already.
bufferPos += size;
}
else
{
// Skipping more bytes than are in the buffer. First skip what we have.
int pos = bufferSize - bufferPos;
// ROK 5/7/2013 Issue #54: should retire all bytes in buffer (bufferSize)
// totalBytesRetired += pos;
totalBytesRetired += bufferSize;
bufferPos = 0;
bufferSize = 0;
// Then skip directly from the InputStream for the rest.
if (pos < size)
{
if (input == null)
{
throw InvalidProtocolBufferException.TruncatedMessage();
}
SkipImpl(size - pos);
totalBytesRetired += size - pos;
}
}
}
/// <summary>
/// Sets currentLimit to (current position) + byteLimit. This is called
/// when descending into a length-delimited embedded message. The previous
/// limit is returned.
/// </summary>
/// <returns>The old limit.</returns>
public static int PushLimit(ref ParserInternalState state, int byteLimit)
{
if (byteLimit < 0)
{
throw InvalidProtocolBufferException.NegativeSize();
}
byteLimit += state.totalBytesRetired + state.bufferPos;
int oldLimit = state.currentLimit;
if (byteLimit > oldLimit)
{
throw InvalidProtocolBufferException.TruncatedMessage();
}
state.currentLimit = byteLimit;
RecomputeBufferSizeAfterLimit(ref state);
return(oldLimit);
}
public void ReadMapItor(Action <CodedInputStream> decoder)
{
int length = ReadLength();
if (length == 0)
{
return;
}
if (recursionDepth >= recursionLimit)
{
throw InvalidProtocolBufferException.RecursionLimitExceeded();
}
int oldLimit = PushLimit(length);
++recursionDepth;
decoder.Invoke(this);
--recursionDepth;
PopLimit(oldLimit);
}
/// <summary>
/// Sets currentLimit to (current position) + byteLimit. This is called
/// when descending into a length-delimited embedded message. The previous
/// limit is returned.
/// </summary>
/// <returns>The old limit.</returns>
internal int PushLimit(int byteLimit)
{
if (byteLimit < 0)
{
throw InvalidProtocolBufferException.NegativeSize();
}
byteLimit += totalBytesRetired + bufferPos;
int oldLimit = currentLimit;
if (byteLimit > oldLimit)
{
throw InvalidProtocolBufferException.TruncatedMessage();
}
currentLimit = byteLimit;
RecomputeBufferSizeAfterLimit();
return(oldLimit);
}
/// <summary>
/// Parses the given bytes using ReadRawVarint32() and ReadRawVarint64() and
/// expects them to fail with an InvalidProtocolBufferException whose
/// description matches the given one.
/// </summary>
private static void AssertReadVarintFailure(InvalidProtocolBufferException expected, byte[] data)
{
CodedInputStream input = new CodedInputStream(data);
var exception = Assert.Throws <InvalidProtocolBufferException>(() => input.ReadRawVarint32());
Assert.AreEqual(expected.Message, exception.Message);
input = new CodedInputStream(data);
exception = Assert.Throws <InvalidProtocolBufferException>(() => input.ReadRawVarint64());
Assert.AreEqual(expected.Message, exception.Message);
AssertReadFromParseContext(new ReadOnlySequence <byte>(data), (ref ParseContext ctx) =>
{
try
{
ctx.ReadUInt32();
Assert.Fail();
}
catch (InvalidProtocolBufferException ex)
{
Assert.AreEqual(expected.Message, ex.Message);
}
}, false);
AssertReadFromParseContext(new ReadOnlySequence <byte>(data), (ref ParseContext ctx) =>
{
try
{
ctx.ReadUInt64();
Assert.Fail();
}
catch (InvalidProtocolBufferException ex)
{
Assert.AreEqual(expected.Message, ex.Message);
}
}, false);
// Make sure we get the same error when reading directly from a Stream.
exception = Assert.Throws <InvalidProtocolBufferException>(() => CodedInputStream.ReadRawVarint32(new MemoryStream(data)));
Assert.AreEqual(expected.Message, exception.Message);
}
/// <summary>
/// Reads and discards <paramref name="size"/> bytes.
/// </summary>
/// <exception cref="InvalidProtocolBufferException">the end of the stream
/// or the current limit was reached</exception>
public static void SkipRawBytes(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state, int size)
{
if (size < 0)
{
throw InvalidProtocolBufferException.NegativeSize();
}
if (state.totalBytesRetired + state.bufferPos + size > state.currentLimit)
{
// Read to the end of the stream anyway.
SkipRawBytes(ref buffer, ref state, state.currentLimit - state.totalBytesRetired - state.bufferPos);
// Then fail.
throw InvalidProtocolBufferException.TruncatedMessage();
}
if (size <= state.bufferSize - state.bufferPos)
{
// We have all the bytes we need already.
state.bufferPos += size;
}
else
{
// Skipping more bytes than are in the buffer. First skip what we have.
int pos = state.bufferSize - state.bufferPos;
state.bufferPos = state.bufferSize;
// TODO: If our segmented buffer is backed by a Stream that is seekable, we could skip the bytes more efficiently
// by simply updating stream's Position property. This used to be supported in the past, but the support was dropped
// because it would make the segmentedBufferHelper more complex. Support can be reintroduced if needed.
state.segmentedBufferHelper.RefillBuffer(ref buffer, ref state, true);
while (size - pos > state.bufferSize)
{
pos += state.bufferSize;
state.bufferPos = state.bufferSize;
state.segmentedBufferHelper.RefillBuffer(ref buffer, ref state, true);
}
state.bufferPos = size - pos;
}
}
/// <summary>
/// Reads an embedded message field value from the stream.
/// </summary>
public void ReadMessage(IMessage builder)
{
int length = ReadLength();
if (recursionDepth >= recursionLimit)
{
throw InvalidProtocolBufferException.RecursionLimitExceeded();
}
int oldLimit = PushLimit(length);
++recursionDepth;
builder.MergeFrom(this);
CheckReadEndOfStreamTag();
// Check that we've read exactly as much data as expected.
if (!ReachedLimit)
{
throw InvalidProtocolBufferException.TruncatedMessage();
}
--recursionDepth;
PopLimit(oldLimit);
}
private static object ParseMapKey(FieldDescriptor field, string keyText)
{
switch (field.FieldType)
{
case FieldType.Bool:
if (keyText == "true")
{
return(true);
}
if (keyText == "false")
{
return(false);
}
throw InvalidProtocolBufferException.OnThrowMessage("Invalid string for bool map key: " + keyText);
case FieldType.String:
return(keyText);
case FieldType.Int32:
case FieldType.SInt32:
case FieldType.SFixed32:
return(ParseNumericString(keyText, int.Parse));
case FieldType.UInt32:
case FieldType.Fixed32:
return(ParseNumericString(keyText, uint.Parse));
case FieldType.Int64:
case FieldType.SInt64:
case FieldType.SFixed64:
return(ParseNumericString(keyText, long.Parse));
case FieldType.UInt64:
case FieldType.Fixed64:
return(ParseNumericString(keyText, ulong.Parse));
default:
throw InvalidProtocolBufferException.OnThrowMessage("Invalid field type for map: " + field.FieldType);
}
}
public static void ReadMessage(ref ParseContext ctx, IMessage message)
{
int length = ParsingPrimitives.ParseLength(ref ctx.buffer, ref ctx.state);
if (ctx.state.recursionDepth >= ctx.state.recursionLimit)
{
throw InvalidProtocolBufferException.RecursionLimitExceeded();
}
int oldLimit = SegmentedBufferHelper.PushLimit(ref ctx.state, length);
++ctx.state.recursionDepth;
ReadRawMessage(ref ctx, message);
CheckReadEndOfStreamTag(ref ctx.state);
// Check that we've read exactly as much data as expected.
if (!SegmentedBufferHelper.IsReachedLimit(ref ctx.state))
{
throw InvalidProtocolBufferException.TruncatedMessage();
}
--ctx.state.recursionDepth;
SegmentedBufferHelper.PopLimit(ref ctx.state, oldLimit);
}
private void SkipGroup()
{
// Note: Currently we expect this to be the way that groups are read. We could put the recursion
// depth changes into the ReadTag method instead, potentially...
recursionDepth++;
if (recursionDepth >= recursionLimit)
{
throw InvalidProtocolBufferException.RecursionLimitExceeded();
}
uint tag;
do
{
tag = ReadTag();
if (tag == 0)
{
throw InvalidProtocolBufferException.TruncatedMessage();
}
// This recursion will allow us to handle nested groups.
SkipLastField();
} while (WireFormat.GetTagWireType(tag) != WireFormat.WireType.EndGroup);
recursionDepth--;
}
/// <summary>
/// Skip a group.
/// </summary>
internal void SkipGroup(uint startGroupTag)
{
// Note: Currently we expect this to be the way that groups are read. We could put the recursion
// depth changes into the ReadTag method instead, potentially...
recursionDepth++;
if (recursionDepth >= recursionLimit)
{
throw InvalidProtocolBufferException.RecursionLimitExceeded();
}
uint tag;
while (true)
{
tag = ReadTag();
if (tag == 0)
{
throw InvalidProtocolBufferException.TruncatedMessage();
}
// Can't call SkipLastField for this case- that would throw.
if (WireFormat.GetTagWireType(tag) == WireFormat.WireType.EndGroup)
{
break;
}
// This recursion will allow us to handle nested groups.
SkipLastField();
}
int startField = WireFormat.GetTagFieldNumber(startGroupTag);
int endField = WireFormat.GetTagFieldNumber(tag);
if (startField != endField)
{
throw new InvalidProtocolBufferException(
string.Format("Mismatched end-group tag. Started with field {0}; ended with field {1}", startField,
endField));
}
recursionDepth--;
}
/// <summary>
/// Skip a group.
/// </summary>
public static void SkipGroup(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state, uint startGroupTag)
{
// Note: Currently we expect this to be the way that groups are read. We could put the recursion
// depth changes into the ReadTag method instead, potentially...
state.recursionDepth++;
if (state.recursionDepth >= state.recursionLimit)
{
throw InvalidProtocolBufferException.RecursionLimitExceeded();
}
uint tag;
while (true)
{
tag = ParsingPrimitives.ParseTag(ref buffer, ref state);
if (tag == 0)
{
throw InvalidProtocolBufferException.TruncatedMessage();
}
// Can't call SkipLastField for this case- that would throw.
if (WireFormat.GetTagWireType(tag) == WireFormat.WireType.EndGroup)
{
break;
}
// This recursion will allow us to handle nested groups.
SkipLastField(ref buffer, ref state);
}
int startField = WireFormat.GetTagFieldNumber(startGroupTag);
int endField = WireFormat.GetTagFieldNumber(tag);
if (startField != endField)
{
throw new InvalidProtocolBufferException(
$"Mismatched end-group tag. Started with field {startField}; ended with field {endField}");
}
state.recursionDepth--;
}
public void ReadVarint()
{
AssertReadVarint(Bytes(0x00), 0);
AssertReadVarint(Bytes(0x01), 1);
AssertReadVarint(Bytes(0x7f), 127);
// 14882
AssertReadVarint(Bytes(0xa2, 0x74), (0x22 << 0) | (0x74 << 7));
// 2961488830
AssertReadVarint(Bytes(0xbe, 0xf7, 0x92, 0x84, 0x0b),
(0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) |
(0x0bL << 28));
// 64-bit
// 7256456126
AssertReadVarint(Bytes(0xbe, 0xf7, 0x92, 0x84, 0x1b),
(0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) |
(0x1bL << 28));
// 41256202580718336
AssertReadVarint(Bytes(0x80, 0xe6, 0xeb, 0x9c, 0xc3, 0xc9, 0xa4, 0x49),
(0x00 << 0) | (0x66 << 7) | (0x6b << 14) | (0x1c << 21) |
(0x43L << 28) | (0x49L << 35) | (0x24L << 42) | (0x49L << 49));
// 11964378330978735131
AssertReadVarint(Bytes(0x9b, 0xa8, 0xf9, 0xc2, 0xbb, 0xd6, 0x80, 0x85, 0xa6, 0x01),
(0x1b << 0) | (0x28 << 7) | (0x79 << 14) | (0x42 << 21) |
(0x3bUL << 28) | (0x56UL << 35) | (0x00UL << 42) |
(0x05UL << 49) | (0x26UL << 56) | (0x01UL << 63));
// Failures
AssertReadVarintFailure(
InvalidProtocolBufferException.MalformedVarint(),
Bytes(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
0x00));
AssertReadVarintFailure(
InvalidProtocolBufferException.TruncatedMessage(),
Bytes(0x80));
}
private void MergeMapField(IMessage message, FieldDescriptor field, JsonTokenizer tokenizer)
{
// Map fields are always objects, even if the values are well-known types: ParseSingleValue handles those.
var token = tokenizer.Next();
if (token.Type != JsonToken.TokenType.StartObject)
{
throw InvalidProtocolBufferException.OnThrowMessage("Expected an object to populate a map");
}
var type = field.MessageType;
var keyField = type.FindFieldByNumber(1);
var valueField = type.FindFieldByNumber(2);
if (keyField == null || valueField == null)
{
throw InvalidProtocolBufferException.OnThrowMessage("Invalid map field: " + field.FullName);
}
IDictionary dictionary = (IDictionary)field.Accessor.GetValue(message);
while (true)
{
token = tokenizer.Next();
if (token.Type == JsonToken.TokenType.EndObject)
{
return;
}
object key = ParseMapKey(keyField, token.StringValue);
object value = ParseSingleValue(valueField, tokenizer);
if (value == null)
{
throw InvalidProtocolBufferException.OnThrowMessage("Map values must not be null");
}
dictionary[key] = value;
}
}
/// <summary>
/// Reads a fixed size of bytes from the input.
/// </summary>
/// <exception cref="InvalidProtocolBufferException">
/// the end of the stream or the current limit was reached
/// </exception>
internal byte[] ReadRawBytes(int size)
{
if (size < 0)
{
throw InvalidProtocolBufferException.NegativeSize();
}
if (totalBytesRetired + bufferPos + size > currentLimit)
{
// Read to the end of the stream (up to the current limit) anyway.
SkipRawBytes(currentLimit - totalBytesRetired - bufferPos);
// Then fail.
throw InvalidProtocolBufferException.TruncatedMessage();
}
if (size <= bufferSize - bufferPos)
{
// We have all the bytes we need already.
byte[] bytes = new byte[size];
ByteArray.Copy(buffer, bufferPos, bytes, 0, size);
bufferPos += size;
return(bytes);
}
else if (size < buffer.Length)
{
// Reading more bytes than are in the buffer, but not an excessive number
// of bytes. We can safely allocate the resulting array ahead of time.
// First copy what we have.
byte[] bytes = new byte[size];
int pos = bufferSize - bufferPos;
ByteArray.Copy(buffer, bufferPos, bytes, 0, pos);
bufferPos = bufferSize;
// We want to use RefillBuffer() and then copy from the buffer into our
// byte array rather than reading directly into our byte array because
// the input may be unbuffered.
RefillBuffer(true);
while (size - pos > bufferSize)
{
Buffer.BlockCopy(buffer, 0, bytes, pos, bufferSize);
pos += bufferSize;
bufferPos = bufferSize;
RefillBuffer(true);
}
ByteArray.Copy(buffer, 0, bytes, pos, size - pos);
bufferPos = size - pos;
return(bytes);
}
else
{
// The size is very large. For security reasons, we can't allocate the
// entire byte array yet. The size comes directly from the input, so a
// maliciously-crafted message could provide a bogus very large size in
// order to trick the app into allocating a lot of memory. We avoid this
// by allocating and reading only a small chunk at a time, so that the
// malicious message must actually *be* extremely large to cause
// problems. Meanwhile, we limit the allowed size of a message elsewhere.
// Remember the buffer markers since we'll have to copy the bytes out of
// it later.
int originalBufferPos = bufferPos;
int originalBufferSize = bufferSize;
// Mark the current buffer consumed.
totalBytesRetired += bufferSize;
bufferPos = 0;
bufferSize = 0;
// Read all the rest of the bytes we need.
int sizeLeft = size - (originalBufferSize - originalBufferPos);
List <byte[]> chunks = new List <byte[]>();
while (sizeLeft > 0)
{
byte[] chunk = new byte[Math.Min(sizeLeft, buffer.Length)];
int pos = 0;
while (pos < chunk.Length)
{
int n = (input == null) ? -1 : input.Read(chunk, pos, chunk.Length - pos);
if (n <= 0)
{
throw InvalidProtocolBufferException.TruncatedMessage();
}
totalBytesRetired += n;
pos += n;
}
sizeLeft -= chunk.Length;
chunks.Add(chunk);
}
// OK, got everything. Now concatenate it all into one buffer.
byte[] bytes = new byte[size];
// Start by copying the leftover bytes from this.buffer.
int newPos = originalBufferSize - originalBufferPos;
ByteArray.Copy(buffer, originalBufferPos, bytes, 0, newPos);
// And now all the chunks.
foreach (byte[] chunk in chunks)
{
Buffer.BlockCopy(chunk, 0, bytes, newPos, chunk.Length);
newPos += chunk.Length;
}
// Done.
return(bytes);
}
}
/// <summary>
/// Merges the given message using data from the given tokenizer. In most cases, the next
/// token should be a "start object" token, but wrapper types and nullity can invalidate
/// that assumption. This is implemented as an LL(1) recursive descent parser over the stream
/// of tokens provided by the tokenizer. This token stream is assumed to be valid JSON, with the
/// tokenizer performing that validation - but not every token stream is valid "protobuf JSON".
/// </summary>
private void Merge(IMessage message, JsonTokenizer tokenizer)
{
if (tokenizer.ObjectDepth > settings.RecursionLimit)
{
throw InvalidProtocolBufferException.JsonRecursionLimitExceeded();
}
if (message.Descriptor.IsWellKnownType)
{
Action <JsonParser, IMessage, JsonTokenizer> handler;
if (WellKnownTypeHandlers.TryGetValue(message.Descriptor.FullName, out handler))
{
handler(this, message, tokenizer);
return;
}
// Well-known types with no special handling continue in the normal way.
}
var token = tokenizer.Next();
if (token.Type != JsonToken.TokenType.StartObject)
{
throw new InvalidProtocolBufferException("Expected an object");
}
var descriptor = message.Descriptor;
var jsonFieldMap = descriptor.Fields.ByJsonName();
// All the oneof fields we've already accounted for - we can only see each of them once.
// The set is created lazily to avoid the overhead of creating a set for every message
// we parsed, when oneofs are relatively rare.
HashSet <OneofDescriptor> seenOneofs = null;
while (true)
{
token = tokenizer.Next();
if (token.Type == JsonToken.TokenType.EndObject)
{
return;
}
if (token.Type != JsonToken.TokenType.Name)
{
throw new InvalidOperationException("Unexpected token type " + token.Type);
}
string name = token.StringValue;
FieldDescriptor field;
if (jsonFieldMap.TryGetValue(name, out field))
{
if (field.ContainingOneof != null)
{
if (seenOneofs == null)
{
seenOneofs = new HashSet <OneofDescriptor>();
}
if (!seenOneofs.Add(field.ContainingOneof))
{
throw new InvalidProtocolBufferException($"Multiple values specified for oneof {field.ContainingOneof.Name}");
}
}
MergeField(message, field, tokenizer);
}
else
{
// TODO: Is this what we want to do? If not, we'll need to skip the value,
// which may be an object or array. (We might want to put code in the tokenizer
// to do that.)
throw new InvalidProtocolBufferException("Unknown field: " + name);
}
}
}
/// <summary>
/// Reads a fixed size of bytes from the input.
/// </summary>
/// <exception cref="InvalidProtocolBufferException">
/// the end of the stream or the current limit was reached
/// </exception>
public static byte[] ReadRawBytes(ref ReadOnlySpan <byte> buffer, ref ParserInternalState state, int size)
{
if (size < 0)
{
throw InvalidProtocolBufferException.NegativeSize();
}
if (state.totalBytesRetired + state.bufferPos + size > state.currentLimit)
{
// Read to the end of the stream (up to the current limit) anyway.
SkipRawBytes(ref buffer, ref state, state.currentLimit - state.totalBytesRetired - state.bufferPos);
// Then fail.
throw InvalidProtocolBufferException.TruncatedMessage();
}
if (size <= state.bufferSize - state.bufferPos)
{
// We have all the bytes we need already.
byte[] bytes = new byte[size];
buffer.Slice(state.bufferPos, size).CopyTo(bytes);
state.bufferPos += size;
return(bytes);
}
else if (size < buffer.Length || size < state.segmentedBufferHelper.TotalLength)
{
// Reading more bytes than are in the buffer, but not an excessive number
// of bytes. We can safely allocate the resulting array ahead of time.
// First copy what we have.
byte[] bytes = new byte[size];
var bytesSpan = new Span <byte>(bytes);
int pos = state.bufferSize - state.bufferPos;
buffer.Slice(state.bufferPos, pos).CopyTo(bytesSpan.Slice(0, pos));
state.bufferPos = state.bufferSize;
// We want to use RefillBuffer() and then copy from the buffer into our
// byte array rather than reading directly into our byte array because
// the input may be unbuffered.
state.segmentedBufferHelper.RefillBuffer(ref buffer, ref state, true);
while (size - pos > state.bufferSize)
{
buffer.Slice(0, state.bufferSize)
.CopyTo(bytesSpan.Slice(pos, state.bufferSize));
pos += state.bufferSize;
state.bufferPos = state.bufferSize;
state.segmentedBufferHelper.RefillBuffer(ref buffer, ref state, true);
}
buffer.Slice(0, size - pos)
.CopyTo(bytesSpan.Slice(pos, size - pos));
state.bufferPos = size - pos;
return(bytes);
}
else
{
// The size is very large. For security reasons, we can't allocate the
// entire byte array yet. The size comes directly from the input, so a
// maliciously-crafted message could provide a bogus very large size in
// order to trick the app into allocating a lot of memory. We avoid this
// by allocating and reading only a small chunk at a time, so that the
// malicious message must actually *be* extremely large to cause
// problems. Meanwhile, we limit the allowed size of a message elsewhere.
List <byte[]> chunks = new List <byte[]>();
int pos = state.bufferSize - state.bufferPos;
byte[] firstChunk = new byte[pos];
buffer.Slice(state.bufferPos, pos).CopyTo(firstChunk);
chunks.Add(firstChunk);
state.bufferPos = state.bufferSize;
// Read all the rest of the bytes we need.
int sizeLeft = size - pos;
while (sizeLeft > 0)
{
state.segmentedBufferHelper.RefillBuffer(ref buffer, ref state, true);
byte[] chunk = new byte[Math.Min(sizeLeft, state.bufferSize)];
buffer.Slice(0, chunk.Length)
.CopyTo(chunk);
state.bufferPos += chunk.Length;
sizeLeft -= chunk.Length;
chunks.Add(chunk);
}
// OK, got everything. Now concatenate it all into one buffer.
byte[] bytes = new byte[size];
int newPos = 0;
foreach (byte[] chunk in chunks)
{
Buffer.BlockCopy(chunk, 0, bytes, newPos, chunk.Length);
newPos += chunk.Length;
}
// Done.
return(bytes);
}
}
/// <summary>
/// Parses the given bytes using ReadRawVarint32() and ReadRawVarint64() and
/// expects them to fail with an InvalidProtocolBufferException whose
/// description matches the given one.
/// </summary>
private static void AssertReadVarintFailure(InvalidProtocolBufferException expected, byte[] data)
{
CodedInputStream input = new CodedInputStream(data);
var exception = Assert.Throws<InvalidProtocolBufferException>(() => input.ReadRawVarint32());
Assert.AreEqual(expected.Message, exception.Message);
input = new CodedInputStream(data);
exception = Assert.Throws<InvalidProtocolBufferException>(() => input.ReadRawVarint64());
Assert.AreEqual(expected.Message, exception.Message);
// Make sure we get the same error when reading directly from a Stream.
exception = Assert.Throws<InvalidProtocolBufferException>(() => CodedInputStream.ReadRawVarint32(new MemoryStream(data)));
Assert.AreEqual(expected.Message, exception.Message);
}
