private static void RunParseWithCustomLexer(IParseEngine parser) { var path = Path.Combine(Directory.GetCurrentDirectory(), "10000.json"); var jsonLexer = new JsonLexer(); using (var stream = File.OpenRead(path)) using (var reader = new StreamReader(stream)) { var tokens = jsonLexer.Lex(reader); foreach (var token in tokens) { if (token.TokenType != JsonLexer.Whitespace) { if (!parser.Pulse(token)) { Assert.Fail($"unable to parse token {token.TokenType} at {token.Position}"); } } } } if (!parser.IsAccepted()) { Assert.Fail("Parse was not accepted"); } }
public void LexingObjectTokensWithoutWhitespacesWorks() { // given var target = new JsonLexer("{}"); // when var obtained = target.NextToken(); // then Assert.That(obtained, Is.Not.Null); Assert.That(obtained.TokenType, Is.EqualTo(JsonTokenType.LEFT_BRACE)); Assert.That(obtained.Value, Is.Null); // and when obtained = target.NextToken(); // then Assert.That(obtained, Is.Not.Null); Assert.That(obtained.TokenType, Is.EqualTo(JsonTokenType.RIGHT_BRACE)); Assert.That(obtained.Value, Is.Null); // and when obtained = target.NextToken(); // then Assert.That(obtained, Is.Null); }
public static XTable Parse(string json) { JsonLexer lexer = new JsonLexer(json); object obj = lexer.NextValue(); return(obj as XTable); }
public void LexingArrayTokesWithoutWhitespaceWorks() { // given var target = new JsonLexer("[]"); // when var obtained = target.NextToken(); // then Assert.That(obtained, Is.Not.Null); Assert.That(obtained.TokenType, Is.EqualTo(JsonTokenType.LEFT_SQUARE_BRACKET)); Assert.That(obtained.Value, Is.Null); // and when obtained = target.NextToken(); // then Assert.That(obtained, Is.Not.Null); Assert.That(obtained.TokenType, Is.EqualTo(JsonTokenType.RIGHT_SQUARE_BRACKET)); Assert.That(obtained.Value, Is.Null); // and when obtained = target.NextToken(); // then Assert.That(obtained, Is.Null); }
public void Lex_Property_Colon_String_Diagnostics_DigitAndEndOfFile2() { string testCase = @"""Entier"": { ""type"": ""int"", ""value"": 123456 }, ""EntierNegatif"": { ""type"": ""int"", ""value"": -123456 }, ""Float"": { ""type"": ""float"", ""value"": 123.3 }, ""Float2"": { ""type"": ""float"", ""value"": 123,3 }, ""Float3"": { ""type"": ""float"", ""value"": 123.356.789 }"; using (JsonLexer jsonLexer = new JsonLexer(testCase)) { int List <SyntaxToken> tokens = jsonLexer.Lex().AsParallel().AsOrdered().ToList(); Assert.IsTrue(!jsonLexer.Diagnostics.Any()); } }
public void JsonLexerShouldReturnFalse() { var input = "false"; var jsonLexer = new JsonLexer(); var tokens = jsonLexer.Lex(input).ToArray(); Assert.AreEqual(1, tokens.Length); Assert.AreEqual(input, tokens[0].Value); }
public void Lex_Property_Colon_String_Diagnostics_MissingEndOfString(string testCase) { using (JsonLexer jsonLexer = new JsonLexer(testCase)) { List <SyntaxToken> tokens = jsonLexer.Lex().AsParallel().AsOrdered().ToList(); Assert.IsTrue(jsonLexer.Diagnostics.Any()); Assert.IsTrue(jsonLexer.Diagnostics.All(d => d.Exception is EndOfFileExtractLiteralException)); } }
public void JsonLexerShouldReturnNull() { var input = "true"; var jsonLexer = new JsonLexer(); var tokens = jsonLexer.Lex(input).ToArray(); Assert.AreEqual(1, tokens.Length); Assert.AreEqual(input, tokens[0].Capture.ToString()); }
public object DeserializeFromJson(ISourceText source, bool caseSensitiveMaps) { var lexer = new JsonLexer(source, throwErrors: true); var parser = new JsonParser(lexer, throwErrors: true, caseSensitiveMaps: caseSensitiveMaps); parser.Parse(); return(parser.ResultContext.ResultObject); }
private static object read(ISourceText source, bool caseSensitiveMaps) { var lexer = new JsonLexer(source, throwErrors: true); var parser = new JsonParser(lexer, throwErrors: true, caseSensitiveMaps: caseSensitiveMaps); parser.Parse(); return(parser.ResultContext.ResultObject); }
public void JsonLexerShouldReadString() { var input = "\"this is a string\""; var jsonLexer = new JsonLexer(); var tokens = jsonLexer.Lex(input).ToArray(); Assert.AreEqual(1, tokens.Length); Assert.AreEqual(input, tokens[0].Capture.ToString()); }
public static JsonParser jsonParserFromString(string input) { var inputStream = new ANTLRStringStream(input); var lexer = new JsonLexer(inputStream); var tokens = new CommonTokenStream(lexer); var parser = new JsonParser(tokens); return(parser); }
public void LexingStringWithTooShortUnicodeEscapeSequenceThrowsAnException() { // given var target = new JsonLexer("\"\\u007\""); // when, then var ex = Assert.Throws <JsonLexerException>(() => target.NextToken()); Assert.That(ex.Message, Is.EqualTo("Invalid unicode escape sequence \"\\u007\"\"")); }
public void LexingStringWithNonHexCharacterInUnicodeEscapeSequenceThrowsAnException() { // given var target = new JsonLexer("\"\\u0048\\u0065\\u006C\\u006C\\u006F\\u0020\\uDDGE\\u0021\""); // when, then var ex = Assert.Throws <JsonLexerException>(() => target.NextToken()); Assert.That(ex.Message, Is.EqualTo("Invalid unicode escape sequence \"\\uDDG\"")); }
public void LexingStringWithLowSurrogateOnlyThrowsAnException() { // given var target = new JsonLexer("\"\\u0048\\u0065\\u006C\\u006C\\u006F\\u0020\\uDD1E\\u0021\""); // when, then var ex = Assert.Throws <JsonLexerException>(() => target.NextToken()); Assert.That(ex.Message, Is.EqualTo("Invalid UTF-16 surrogate pair \"\\uDD1E\"")); }
public void LexingMinusSignWithoutSubsequentDigitsThrowsException() { // given var target = new JsonLexer("-"); // when, then var ex = Assert.Throws <JsonLexerException>(() => target.NextToken()); Assert.That(ex.Message, Is.EqualTo("Invalid number literal \"-\"")); }
public void LexingUnterminatedStringAfterExcapesequenceThrowsAnException() { // given var target = new JsonLexer("\"foo \\"); // when, then var ex = Assert.Throws <JsonLexerException>(() => target.NextToken()); Assert.That(ex.Message, Is.EqualTo("Unterminated string literal \"foo \"")); }
public void LexingStringWithInvalidEscapeSequenceThrowsException() { // given var target = new JsonLexer("\"Hello \\a World!\'"); // when, then var ex = Assert.Throws <JsonLexerException>(() => target.NextToken()); Assert.That(ex.Message, Is.EqualTo("Invalid escape sequence \"\\a\"")); }
public void LexingNumberWithDecimalSeparatorImmediatelyFollowedByUpperCaseExponentThrowsException() { // given var target = new JsonLexer("1.E-5"); // when, then var ex = Assert.Throws <JsonLexerException>(() => target.NextToken()); Assert.That(ex.Message, Is.EqualTo("Invalid number literal \"1.E-5\"")); }
public void LexingNumberWithExponentAndNoDigitsThrowsException() { // given var target = new JsonLexer("1e"); // when, then var ex = Assert.Throws <JsonLexerException>(() => target.NextToken()); Assert.That(ex.Message, Is.EqualTo("Invalid number literal \"1e\"")); }
public void LexingNumberWithDecimalSeparatorAndNoSubsequentDigitsThrowsException() { // given var target = new JsonLexer("1234."); // when, then var ex = Assert.Throws <JsonLexerException>(() => target.NextToken()); Assert.That(ex.Message, Is.EqualTo("Invalid number literal \"1234.\"")); }
public void LexingNumberWithLeadingZeroThrowsException() { // given var target = new JsonLexer("01234"); // when, then var ex = Assert.Throws <JsonLexerException>(() => target.NextToken()); Assert.That(ex.Message, Is.EqualTo("Invalid number literal \"01234\"")); }
public void LexingIntegerNumberWithLeadingPlusThrowsException() { // given var target = new JsonLexer("+42"); // when, then var ex = Assert.Throws <JsonLexerException>(() => target.NextToken()); Assert.That(ex.Message, Is.EqualTo("Unexpected literal \"+42\"")); }
public void LexingStringWithCharactersThatMustBeEscapedButAreNotThrowsAnException() { // given var target = new JsonLexer("\"\n\""); // when, then var ex = Assert.Throws <JsonLexerException>(() => target.NextToken()); Assert.That(ex.Message, Is.EqualTo("Invalid control character \"\\u000A\"")); }
public void LexingNumberWithUpperCaseExponentFollowedByMinusAndNoDigitsThrowsException() { // given var target = new JsonLexer("2E-"); // when var ex = Assert.Throws <JsonLexerException>(() => target.NextToken()); Assert.That(ex.Message, Is.EqualTo("Invalid number literal \"2E-\"")); }
public void LexingNullLiteralWithWrongCasingThrowsAnError() { // given var target = new JsonLexer("nUll"); // when, then var ex = Assert.Throws <JsonLexerException>(() => target.NextToken()); Assert.That(ex.Message, Is.EqualTo("Unexpected literal \"nUll\"")); }
public void JsonLexerShouldReadSingleDigit() { var input = "1"; var jsonLexer = new JsonLexer(); var tokens = jsonLexer.Lex(input).ToArray(); Assert.AreEqual(1, tokens.Length); Assert.AreEqual(input, tokens[0].Capture.ToString()); Assert.AreEqual(0, tokens[0].Position); }
public void LexingStringWithWhitespaceOnlyReturnsNullAsNextToken() { // given var target = new JsonLexer(" \r\n\t"); // when var obtained = target.NextToken(); // then Assert.That(obtained, Is.Null); }
public void Lex_Property_Colon_String_Diagnostics_LettersWhiteSpaceAndEndOfFile(string testCase) { using (JsonLexer jsonLexer = new JsonLexer(testCase)) { List <SyntaxToken> tokens = jsonLexer.Lex().AsParallel().AsOrdered().ToList(); Assert.IsTrue(!jsonLexer.Diagnostics.Any()); Assert.AreEqual(tokens[0].Kind, SyntaxKind.Literal); Assert.AreEqual(tokens[1].Kind, SyntaxKind.Whitespace); Assert.AreEqual(tokens.Last().Kind, SyntaxKind.EndOfFile); } }
public void LexingEmptyStringReturnsNullAsNextToken() { // given var target = new JsonLexer(""); // when var obtained = target.NextToken(); // then Assert.That(obtained, Is.Null); }
/// <summary> /// Parses the specified JSON input. /// </summary> public Node Parse(String input) { Assume.NotNull(input, nameof(input)); var stream = new AntlrInputStream(input); ITokenSource lexer = new JsonLexer(stream); ITokenStream tokens = new CommonTokenStream(lexer); var parser = new JsonParser(tokens) { BuildParseTree = true }; var tree = parser.start(); var treeFactory = new StandardJsonSyntaxTreeFactory(); var syntaxBuilder = new StandardJsonSyntaxTreeBuilder(treeFactory); return syntaxBuilder.Visit(input, tree); }