public void ObjectDepth() { string json = "{ \"foo\": { \"x\": 1, \"y\": [ 0 ] } }"; var tokenizer = JsonTokenizer.FromTextReader(new StringReader(json)); // If we had more tests like this, I'd introduce a helper method... but for one test, it's not worth it. Assert.AreEqual(0, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); Assert.AreEqual(1, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.Name("foo"), tokenizer.Next()); Assert.AreEqual(1, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); Assert.AreEqual(2, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.Name("x"), tokenizer.Next()); Assert.AreEqual(2, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.Value(1), tokenizer.Next()); Assert.AreEqual(2, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.Name("y"), tokenizer.Next()); Assert.AreEqual(2, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.StartArray, tokenizer.Next()); Assert.AreEqual(2, tokenizer.ObjectDepth); // Depth hasn't changed in array Assert.AreEqual(JsonToken.Value(0), tokenizer.Next()); Assert.AreEqual(2, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.EndArray, tokenizer.Next()); Assert.AreEqual(2, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.EndObject, tokenizer.Next()); Assert.AreEqual(1, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.EndObject, tokenizer.Next()); Assert.AreEqual(0, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.EndDocument, tokenizer.Next()); Assert.AreEqual(0, tokenizer.ObjectDepth); }
private void MergeField(IMessage message, FieldDescriptor field, JsonTokenizer tokenizer) { var token = tokenizer.Next(); if (token.Type == JsonToken.TokenType.Null) { // Note: different from Java API, which just ignores it. // TODO: Bring it more in line? Discuss... field.Accessor.Clear(message); return; } tokenizer.PushBack(token); if (field.IsMap) { MergeMapField(message, field, tokenizer); } else if (field.IsRepeated) { MergeRepeatedField(message, field, tokenizer); } else { var value = ParseSingleValue(field, tokenizer); field.Accessor.SetValue(message, value); } }
private void MergeRepeatedField(IMessage message, FieldDescriptor field, JsonTokenizer tokenizer) { var token = tokenizer.Next(); if (token.Type != JsonToken.TokenType.StartArray) { throw new InvalidProtocolBufferException("Repeated field value was not an array. Token type: " + token.Type); } IList list = (IList)field.Accessor.GetValue(message); while (true) { token = tokenizer.Next(); if (token.Type == JsonToken.TokenType.EndArray) { return; } tokenizer.PushBack(token); if (token.Type == JsonToken.TokenType.Null) { throw new InvalidProtocolBufferException("Repeated field elements cannot be null"); } list.Add(ParseSingleValue(field, tokenizer)); } }
private void MergeMapField(IMessage message, FieldDescriptor field, JsonTokenizer tokenizer) { // Map fields are always objects, even if the values are well-known types: ParseSingleValue handles those. var token = tokenizer.Next(); if (token.Type != JsonToken.TokenType.StartObject) { throw new InvalidProtocolBufferException("Expected an object to populate a map"); } var type = field.MessageType; var keyField = type.FindFieldByNumber(1); var valueField = type.FindFieldByNumber(2); if (keyField == null || valueField == null) { throw new InvalidProtocolBufferException("Invalid map field: " + field.FullName); } IDictionary dictionary = (IDictionary)field.Accessor.GetValue(message); while (true) { token = tokenizer.Next(); if (token.Type == JsonToken.TokenType.EndObject) { return; } object key = ParseMapKey(keyField, token.StringValue); object value = ParseSingleValue(valueField, tokenizer); // TODO: Null handling dictionary[key] = value; } }
private void MergeField(IMessage message, FieldDescriptor field, JsonTokenizer tokenizer) { var token = tokenizer.Next(); if (token.Type == JsonToken.TokenType.Null) { // Clear the field if we see a null token, unless it's for a singular field of type // google.protobuf.Value. // Note: different from Java API, which just ignores it. // TODO: Bring it more in line? Discuss... if (field.IsMap || field.IsRepeated || !IsGoogleProtobufValueField(field)) { field.Accessor.Clear(message); return; } } tokenizer.PushBack(token); if (field.IsMap) { MergeMapField(message, field, tokenizer); } else if (field.IsRepeated) { MergeRepeatedField(message, field, tokenizer); } else { var value = ParseSingleValue(field, tokenizer); field.Accessor.SetValue(message, value); } }
private void MergeAny(IMessage message, JsonTokenizer tokenizer) { // Record the token stream until we see the @type property. At that point, we can take the value, consult // the type registry for the relevant message, and replay the stream, omitting the @type property. var tokens = new List <JsonToken>(); var token = tokenizer.Next(); if (token.Type != JsonToken.TokenType.StartObject) { throw new InvalidProtocolBufferException("Expected object value for Any"); } int typeUrlObjectDepth = tokenizer.ObjectDepth; // The check for the property depth protects us from nested Any values which occur before the type URL // for *this* Any. while (token.Type != JsonToken.TokenType.Name || token.StringValue != JsonFormatter.AnyTypeUrlField || tokenizer.ObjectDepth != typeUrlObjectDepth) { tokens.Add(token); token = tokenizer.Next(); } // Don't add the @type property or its value to the recorded token list token = tokenizer.Next(); if (token.Type != JsonToken.TokenType.StringValue) { throw new InvalidProtocolBufferException("Expected string value for Any.@type"); } string typeUrl = token.StringValue; string typeName = JsonFormatter.GetTypeName(typeUrl); MessageDescriptor descriptor = settings.TypeRegistry.Find(typeName); if (descriptor == null) { throw new InvalidOperationException($"Type registry has no descriptor for type name '{typeName}'"); } // Now replay the token stream we've already read and anything that remains of the object, just parsing it // as normal. Our original tokenizer should end up at the end of the object. var replay = JsonTokenizer.FromReplayedTokens(tokens, tokenizer); var body = descriptor.Parser.CreateTemplate(); if (descriptor.IsWellKnownType) { MergeWellKnownTypeAnyBody(body, replay); } else { Merge(body, replay); } var data = body.ToByteString(); // Now that we have the message data, we can pack it into an Any (the message received as a parameter). message.Descriptor.Fields[Any.TypeUrlFieldNumber].Accessor.SetValue(message, typeUrl); message.Descriptor.Fields[Any.ValueFieldNumber].Accessor.SetValue(message, data); }
public void NextAfterEndDocumentThrows() { var tokenizer = JsonTokenizer.FromTextReader(new StringReader("null")); Assert.AreEqual(JsonToken.Null, tokenizer.Next()); Assert.AreEqual(JsonToken.EndDocument, tokenizer.Next()); Assert.Throws <InvalidOperationException>(() => tokenizer.Next()); }
public void SkipValue(string json) { var tokenizer = JsonTokenizer.FromTextReader(new StringReader(json.Replace('\'', '"'))); Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); Assert.AreEqual("skip", tokenizer.Next().StringValue); tokenizer.SkipValue(); Assert.AreEqual("next", tokenizer.Next().StringValue); }
public void CanPushBackEndDocument() { var tokenizer = new JsonTokenizer(new StringReader("null")); Assert.AreEqual(JsonToken.Null, tokenizer.Next()); Assert.AreEqual(JsonToken.EndDocument, tokenizer.Next()); tokenizer.PushBack(JsonToken.EndDocument); Assert.AreEqual(JsonToken.EndDocument, tokenizer.Next()); Assert.Throws <InvalidOperationException>(() => tokenizer.Next()); }
/// <summary> /// Parses JSON read from <paramref name="jsonReader"/> and merges the information into the given message. /// </summary> /// <param name="message">The message to merge the JSON information into.</param> /// <param name="jsonReader">Reader providing the JSON to parse.</param> internal void Merge(IMessage message, TextReader jsonReader) { var tokenizer = JsonTokenizer.FromTextReader(jsonReader); Merge(message, tokenizer); var lastToken = tokenizer.Next(); if (lastToken != JsonToken.EndDocument) { throw new InvalidProtocolBufferException("Expected end of JSON after object"); } }
public void InvalidStructure(string json, int expectedValidTokens) { // Note: we don't test that the earlier tokens are exactly as expected, // partly because that's hard to parameterize. var reader = new StringReader(json.Replace('\'', '"')); var tokenizer = JsonTokenizer.FromTextReader(reader); for (int i = 0; i < expectedValidTokens; i++) { Assert.IsNotNull(tokenizer.Next()); } Assert.Throws <InvalidJsonException>(() => tokenizer.Next()); }
/// <summary> /// Merges the given message using data from the given tokenizer. In most cases, the next /// token should be a "start object" token, but wrapper types and nullity can invalidate /// that assumption. This is implemented as an LL(1) recursive descent parser over the stream /// of tokens provided by the tokenizer. This token stream is assumed to be valid JSON, with the /// tokenizer performing that validation - but not every token stream is valid "protobuf JSON". /// </summary> private void Merge(IMessage message, JsonTokenizer tokenizer) { if (message.Descriptor.IsWellKnownType) { Action <JsonParser, IMessage, JsonTokenizer> handler; if (WellKnownTypeHandlers.TryGetValue(message.Descriptor.FullName, out handler)) { handler(this, message, tokenizer); return; } // Well-known types with no special handling continue in the normal way. } var token = tokenizer.Next(); if (token.Type != JsonToken.TokenType.StartObject) { throw new InvalidProtocolBufferException("Expected an object"); } var descriptor = message.Descriptor; // TODO: Make this more efficient, e.g. by building it once in the descriptor. // Additionally, we need to consider whether to parse field names in their original proto form, // and any overrides in the descriptor. But yes, all of this should be in the descriptor somehow... // the descriptor can expose the dictionary. var jsonFieldMap = descriptor.Fields.InDeclarationOrder().ToDictionary(field => JsonFormatter.ToCamelCase(field.Name)); while (true) { token = tokenizer.Next(); if (token.Type == JsonToken.TokenType.EndObject) { return; } if (token.Type != JsonToken.TokenType.Name) { throw new InvalidOperationException("Unexpected token type " + token.Type); } string name = token.StringValue; FieldDescriptor field; if (jsonFieldMap.TryGetValue(name, out field)) { MergeField(message, field, tokenizer); } else { // TODO: Is this what we want to do? If not, we'll need to skip the value, // which may be an object or array. (We might want to put code in the tokenizer // to do that.) throw new InvalidProtocolBufferException("Unknown field: " + name); } } }
/// <summary> /// Merges the given message using data from the given tokenizer. In most cases, the next /// token should be a "start object" token, but wrapper types and nullity can invalidate /// that assumption. This is implemented as an LL(1) recursive descent parser over the stream /// of tokens provided by the tokenizer. This token stream is assumed to be valid JSON, with the /// tokenizer performing that validation - but not every token stream is valid "protobuf JSON". /// </summary> private void Merge(IMessage message, JsonTokenizer tokenizer) { if (tokenizer.ObjectDepth > settings.RecursionLimit) { throw InvalidProtocolBufferException.JsonRecursionLimitExceeded(); } if (message.Descriptor.IsWellKnownType) { Action <JsonParser, IMessage, JsonTokenizer> handler; if (WellKnownTypeHandlers.TryGetValue(message.Descriptor.FullName, out handler)) { handler(this, message, tokenizer); return; } // Well-known types with no special handling continue in the normal way. } var token = tokenizer.Next(); if (token.Type != JsonToken.TokenType.StartObject) { throw new InvalidProtocolBufferException("Expected an object"); } var descriptor = message.Descriptor; var jsonFieldMap = descriptor.Fields.ByJsonName(); while (true) { token = tokenizer.Next(); if (token.Type == JsonToken.TokenType.EndObject) { return; } if (token.Type != JsonToken.TokenType.Name) { throw new InvalidOperationException("Unexpected token type " + token.Type); } string name = token.StringValue; FieldDescriptor field; if (jsonFieldMap.TryGetValue(name, out field)) { MergeField(message, field, tokenizer); } else { // TODO: Is this what we want to do? If not, we'll need to skip the value, // which may be an object or array. (We might want to put code in the tokenizer // to do that.) throw new InvalidProtocolBufferException("Unknown field: " + name); } } }
private void MergeStruct(IMessage message, JsonTokenizer tokenizer) { var token = tokenizer.Next(); if (token.Type != JsonToken.TokenType.StartObject) { throw new InvalidProtocolBufferException("Expected object value for Struct"); } tokenizer.PushBack(token); var field = message.Descriptor.Fields[Struct.FieldsFieldNumber]; MergeMapField(message, field, tokenizer); }
private void MergeStructValue(IMessage message, JsonTokenizer tokenizer) { var firstToken = tokenizer.Next(); var fields = message.Descriptor.Fields; switch (firstToken.Type) { case JsonToken.TokenType.Null: fields[Value.NullValueFieldNumber].Accessor.SetValue(message, 0); return; case JsonToken.TokenType.StringValue: fields[Value.StringValueFieldNumber].Accessor.SetValue(message, firstToken.StringValue); return; case JsonToken.TokenType.Number: fields[Value.NumberValueFieldNumber].Accessor.SetValue(message, firstToken.NumberValue); return; case JsonToken.TokenType.False: case JsonToken.TokenType.True: fields[Value.BoolValueFieldNumber].Accessor.SetValue(message, firstToken.Type == JsonToken.TokenType.True); return; case JsonToken.TokenType.StartObject: { var field = fields[Value.StructValueFieldNumber]; var structMessage = NewMessageForField(field); tokenizer.PushBack(firstToken); Merge(structMessage, tokenizer); field.Accessor.SetValue(message, structMessage); return; } case JsonToken.TokenType.StartArray: { var field = fields[Value.ListValueFieldNumber]; var list = NewMessageForField(field); tokenizer.PushBack(firstToken); Merge(list, tokenizer); field.Accessor.SetValue(message, list); return; } default: throw new InvalidOperationException("Unexpected token type: " + firstToken.Type); } }
private static void AssertThrowsAfter(string json, params JsonToken[] expectedTokens) { var reader = new StringReader(json); var tokenizer = JsonTokenizer.FromTextReader(reader); for (int i = 0; i < expectedTokens.Length; i++) { var actualToken = tokenizer.Next(); if (actualToken == JsonToken.EndDocument) { Assert.Fail("Expected {0} but reached end of document", expectedTokens[i]); } Assert.AreEqual(expectedTokens[i], actualToken); } Assert.Throws <InvalidJsonException>(() => tokenizer.Next()); }
// Well-known types end up in a property called "value" in the JSON. As there's no longer a @type property // in the given JSON token stream, we should *only* have tokens of start-object, name("value"), the value // itself, and then end-object. private void MergeWellKnownTypeAnyBody(IMessage body, JsonTokenizer tokenizer) { var token = tokenizer.Next(); // Definitely start-object; checked in previous method token = tokenizer.Next(); // TODO: What about an absent Int32Value, for example? if (token.Type != JsonToken.TokenType.Name || token.StringValue != JsonFormatter.AnyWellKnownTypeValueField) { throw new InvalidProtocolBufferException($"Expected '{JsonFormatter.AnyWellKnownTypeValueField}' property for well-known type Any body"); } Merge(body, tokenizer); token = tokenizer.Next(); if (token.Type != JsonToken.TokenType.EndObject) { throw new InvalidProtocolBufferException($"Expected end-object token after @type/value for well-known type"); } }
/// <summary> /// Asserts that the specified JSON is tokenized into the given sequence of tokens. /// Unlike <see cref="AssertTokens(string, JsonToken[])"/>, this does not perform any character /// replacement on the specified JSON, and should be used when the text contains apostrophes which /// are expected to be used *as* apostrophes. The "end document" token is not specified in the list of /// expected tokens, but is implicit. /// </summary> private static void AssertTokensNoReplacement(string json, params JsonToken[] expectedTokens) { var reader = new StringReader(json); var tokenizer = JsonTokenizer.FromTextReader(reader); for (int i = 0; i < expectedTokens.Length; i++) { var actualToken = tokenizer.Next(); if (actualToken == JsonToken.EndDocument) { Assert.Fail("Expected {0} but reached end of token stream", expectedTokens[i]); } Assert.AreEqual(expectedTokens[i], actualToken); } var finalToken = tokenizer.Next(); if (finalToken != JsonToken.EndDocument) { Assert.Fail("Expected token stream to be exhausted; received {0}", finalToken); } }
public void ObjectDepth_WithPushBack() { string json = "{}"; var tokenizer = JsonTokenizer.FromTextReader(new StringReader(json)); Assert.AreEqual(0, tokenizer.ObjectDepth); var token = tokenizer.Next(); Assert.AreEqual(1, tokenizer.ObjectDepth); // When we push back a "start object", we should effectively be back to the previous depth. tokenizer.PushBack(token); Assert.AreEqual(0, tokenizer.ObjectDepth); // Read the same token again, and get back to depth 1 token = tokenizer.Next(); Assert.AreEqual(1, tokenizer.ObjectDepth); // Now the same in reverse, with EndObject token = tokenizer.Next(); Assert.AreEqual(0, tokenizer.ObjectDepth); tokenizer.PushBack(token); Assert.AreEqual(1, tokenizer.ObjectDepth); tokenizer.Next(); Assert.AreEqual(0, tokenizer.ObjectDepth); }
private object ParseSingleValue(FieldDescriptor field, JsonTokenizer tokenizer) { var token = tokenizer.Next(); if (token.Type == JsonToken.TokenType.Null) { if (field.FieldType == FieldType.Message && field.MessageType.FullName == Value.Descriptor.FullName) { return(new Value { NullValue = NullValue.NULL_VALUE }); } return(null); } var fieldType = field.FieldType; if (fieldType == FieldType.Message) { // Parse wrapper types as their constituent types. // TODO: What does this mean for null? if (field.MessageType.IsWrapperType) { field = field.MessageType.Fields[WrappersReflection.WrapperValueFieldNumber]; fieldType = field.FieldType; } else { // TODO: Merge the current value in message? (Public API currently doesn't make this relevant as we don't expose merging.) tokenizer.PushBack(token); IMessage subMessage = NewMessageForField(field); Merge(subMessage, tokenizer); return(subMessage); } } switch (token.Type) { case JsonToken.TokenType.True: case JsonToken.TokenType.False: if (fieldType == FieldType.Bool) { return(token.Type == JsonToken.TokenType.True); } // Fall through to "we don't support this type for this case"; could duplicate the behaviour of the default // case instead, but this way we'd only need to change one place. goto default; case JsonToken.TokenType.StringValue: return(ParseSingleStringValue(field, token.StringValue)); // Note: not passing the number value itself here, as we may end up storing the string value in the token too. case JsonToken.TokenType.Number: return(ParseSingleNumberValue(field, token)); case JsonToken.TokenType.Null: throw new NotImplementedException("Haven't worked out what to do for null yet"); default: throw new InvalidProtocolBufferException("Unsupported JSON token type " + token.Type + " for field type " + fieldType); } }
internal JsonReplayTokenizer(List <JsonToken> tokens, JsonTokenizer nextTokenizer) { this.tokens = tokens; this.nextTokenizer = nextTokenizer; }
/// <summary> /// Creates a tokenizer that first replays the given list of tokens, then continues reading /// from another tokenizer. Note that if the returned tokenizer is "pushed back", that does not push back /// on the continuation tokenizer, or vice versa. Care should be taken when using this method - it was /// created for the sake of Any parsing. /// </summary> internal static JsonTokenizer FromReplayedTokens(List <JsonToken> tokens, JsonTokenizer continuation) { return(new JsonReplayTokenizer(tokens, continuation)); }
/// <summary> /// Merges the given message using data from the given tokenizer. In most cases, the next /// token should be a "start object" token, but wrapper types and nullity can invalidate /// that assumption. This is implemented as an LL(1) recursive descent parser over the stream /// of tokens provided by the tokenizer. This token stream is assumed to be valid JSON, with the /// tokenizer performing that validation - but not every token stream is valid "protobuf JSON". /// </summary> private void Merge(IMessage message, JsonTokenizer tokenizer) { if (tokenizer.ObjectDepth > settings.RecursionLimit) { throw InvalidProtocolBufferException.JsonRecursionLimitExceeded(); } if (message.Descriptor.IsWellKnownType) { Action <JsonParser, IMessage, JsonTokenizer> handler; if (WellKnownTypeHandlers.TryGetValue(message.Descriptor.FullName, out handler)) { handler(this, message, tokenizer); return; } // Well-known types with no special handling continue in the normal way. } var token = tokenizer.Next(); if (token.Type != JsonToken.TokenType.StartObject) { throw new InvalidProtocolBufferException("Expected an object"); } var descriptor = message.Descriptor; var jsonFieldMap = descriptor.Fields.ByJsonName(); // All the oneof fields we've already accounted for - we can only see each of them once. // The set is created lazily to avoid the overhead of creating a set for every message // we parsed, when oneofs are relatively rare. HashSet <OneofDescriptor> seenOneofs = null; while (true) { token = tokenizer.Next(); if (token.Type == JsonToken.TokenType.EndObject) { return; } if (token.Type != JsonToken.TokenType.Name) { throw new InvalidOperationException("Unexpected token type " + token.Type); } string name = token.StringValue; FieldDescriptor field; if (jsonFieldMap.TryGetValue(name, out field)) { if (field.ContainingOneof != null) { if (seenOneofs == null) { seenOneofs = new HashSet <OneofDescriptor>(); } if (!seenOneofs.Add(field.ContainingOneof)) { throw new InvalidProtocolBufferException($"Multiple values specified for oneof {field.ContainingOneof.Name}"); } } MergeField(message, field, tokenizer); } else { // TODO: Is this what we want to do? If not, we'll need to skip the value, // which may be an object or array. (We might want to put code in the tokenizer // to do that.) throw new InvalidProtocolBufferException("Unknown field: " + name); } } }
/// <summary> /// Creates a tokenizer that first replays the given list of tokens, then continues reading /// from another tokenizer. Note that if the returned tokenizer is "pushed back", that does not push back /// on the continuation tokenizer, or vice versa. Care should be taken when using this method - it was /// created for the sake of Any parsing. /// </summary> internal static JsonTokenizer FromReplayedTokens(IList<JsonToken> tokens, JsonTokenizer continuation) { return new JsonReplayTokenizer(tokens, continuation); }
internal JsonReplayTokenizer(IList<JsonToken> tokens, JsonTokenizer nextTokenizer) { this.tokens = tokens; this.nextTokenizer = nextTokenizer; }
private static void AssertThrowsAfter(string json, params JsonToken[] expectedTokens) { var reader = new StringReader(json); var tokenizer = new JsonTokenizer(reader); for (int i = 0; i < expectedTokens.Length; i++) { var actualToken = tokenizer.Next(); if (actualToken == JsonToken.EndDocument) { Assert.Fail("Expected {0} but reached end of document", expectedTokens[i]); } Assert.AreEqual(expectedTokens[i], actualToken); } Assert.Throws<InvalidProtocolBufferException>(() => tokenizer.Next()); }
/// <summary> /// Asserts that the specified JSON is tokenized into the given sequence of tokens. /// Unlike <see cref="AssertTokens(string, JsonToken[])"/>, this does not perform any character /// replacement on the specified JSON, and should be used when the text contains apostrophes which /// are expected to be used *as* apostrophes. The "end document" token is not specified in the list of /// expected tokens, but is implicit. /// </summary> private static void AssertTokensNoReplacement(string json, params JsonToken[] expectedTokens) { var reader = new StringReader(json); var tokenizer = new JsonTokenizer(reader); for (int i = 0; i < expectedTokens.Length; i++) { var actualToken = tokenizer.Next(); if (actualToken == JsonToken.EndDocument) { Assert.Fail("Expected {0} but reached end of token stream", expectedTokens[i]); } Assert.AreEqual(expectedTokens[i], actualToken); } var finalToken = tokenizer.Next(); if (finalToken != JsonToken.EndDocument) { Assert.Fail("Expected token stream to be exhausted; received {0}", finalToken); } }
public void CanPushBackEndDocument() { var tokenizer = new JsonTokenizer(new StringReader("null")); Assert.AreEqual(JsonToken.Null, tokenizer.Next()); Assert.AreEqual(JsonToken.EndDocument, tokenizer.Next()); tokenizer.PushBack(JsonToken.EndDocument); Assert.AreEqual(JsonToken.EndDocument, tokenizer.Next()); Assert.Throws<InvalidOperationException>(() => tokenizer.Next()); }
public void InvalidStructure(string json, int expectedValidTokens) { // Note: we don't test that the earlier tokens are exactly as expected, // partly because that's hard to parameterize. var reader = new StringReader(json.Replace('\'', '"')); var tokenizer = new JsonTokenizer(reader); for (int i = 0; i < expectedValidTokens; i++) { Assert.IsNotNull(tokenizer.Next()); } Assert.Throws<InvalidProtocolBufferException>(() => tokenizer.Next()); }
public void ObjectDepth() { string json = "{ \"foo\": { \"x\": 1, \"y\": [ 0 ] } }"; var tokenizer = new JsonTokenizer(new StringReader(json)); // If we had more tests like this, I'd introduce a helper method... but for one test, it's not worth it. Assert.AreEqual(0, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); Assert.AreEqual(1, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.Name("foo"), tokenizer.Next()); Assert.AreEqual(1, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); Assert.AreEqual(2, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.Name("x"), tokenizer.Next()); Assert.AreEqual(2, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.Value(1), tokenizer.Next()); Assert.AreEqual(2, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.Name("y"), tokenizer.Next()); Assert.AreEqual(2, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.StartArray, tokenizer.Next()); Assert.AreEqual(2, tokenizer.ObjectDepth); // Depth hasn't changed in array Assert.AreEqual(JsonToken.Value(0), tokenizer.Next()); Assert.AreEqual(2, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.EndArray, tokenizer.Next()); Assert.AreEqual(2, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.EndObject, tokenizer.Next()); Assert.AreEqual(1, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.EndObject, tokenizer.Next()); Assert.AreEqual(0, tokenizer.ObjectDepth); Assert.AreEqual(JsonToken.EndDocument, tokenizer.Next()); Assert.AreEqual(0, tokenizer.ObjectDepth); }
public JsonReplayTokenizer(IList <JsonToken> tokens, JsonTokenizer nextTokenizer) { this.tokens = tokens; this.nextTokenizer = nextTokenizer; }
// Convenience method to avoid having to repeat the same code multiple times in the above // dictionary initialization. private static void MergeWrapperField(JsonParser parser, IMessage message, JsonTokenizer tokenizer) { parser.MergeField(message, message.Descriptor.Fields[WrappersReflection.WrapperValueFieldNumber], tokenizer); }
public void ObjectDepth_WithPushBack() { string json = "{}"; var tokenizer = new JsonTokenizer(new StringReader(json)); Assert.AreEqual(0, tokenizer.ObjectDepth); var token = tokenizer.Next(); Assert.AreEqual(1, tokenizer.ObjectDepth); // When we push back a "start object", we should effectively be back to the previous depth. tokenizer.PushBack(token); Assert.AreEqual(0, tokenizer.ObjectDepth); // Read the same token again, and get back to depth 1 token = tokenizer.Next(); Assert.AreEqual(1, tokenizer.ObjectDepth); // Now the same in reverse, with EndObject token = tokenizer.Next(); Assert.AreEqual(0, tokenizer.ObjectDepth); tokenizer.PushBack(token); Assert.AreEqual(1, tokenizer.ObjectDepth); tokenizer.Next(); Assert.AreEqual(0, tokenizer.ObjectDepth); }