private static void RunParseWithCustomLexer(IParseEngine parser)
{
var path = Path.Combine(Directory.GetCurrentDirectory(), "10000.json");
var jsonLexer = new JsonLexer();
using (var stream = File.OpenRead(path))
using (var reader = new StreamReader(stream))
{
var tokens = jsonLexer.Lex(reader);
foreach (var token in tokens)
{
if (token.TokenType != JsonLexer.Whitespace)
{
if (!parser.Pulse(token))
{
Assert.Fail($"unable to parse token {token.TokenType} at {token.Position}");
}
}
}
}
if (!parser.IsAccepted())
{
Assert.Fail("Parse was not accepted");
}
}
public void LexingObjectTokensWithoutWhitespacesWorks()
{
// given
var target = new JsonLexer("{}");
// when
var obtained = target.NextToken();
// then
Assert.That(obtained, Is.Not.Null);
Assert.That(obtained.TokenType, Is.EqualTo(JsonTokenType.LEFT_BRACE));
Assert.That(obtained.Value, Is.Null);
// and when
obtained = target.NextToken();
// then
Assert.That(obtained, Is.Not.Null);
Assert.That(obtained.TokenType, Is.EqualTo(JsonTokenType.RIGHT_BRACE));
Assert.That(obtained.Value, Is.Null);
// and when
obtained = target.NextToken();
// then
Assert.That(obtained, Is.Null);
}
public static XTable Parse(string json)
{
JsonLexer lexer = new JsonLexer(json);
object obj = lexer.NextValue();
return(obj as XTable);
}
public void LexingArrayTokesWithoutWhitespaceWorks()
{
// given
var target = new JsonLexer("[]");
// when
var obtained = target.NextToken();
// then
Assert.That(obtained, Is.Not.Null);
Assert.That(obtained.TokenType, Is.EqualTo(JsonTokenType.LEFT_SQUARE_BRACKET));
Assert.That(obtained.Value, Is.Null);
// and when
obtained = target.NextToken();
// then
Assert.That(obtained, Is.Not.Null);
Assert.That(obtained.TokenType, Is.EqualTo(JsonTokenType.RIGHT_SQUARE_BRACKET));
Assert.That(obtained.Value, Is.Null);
// and when
obtained = target.NextToken();
// then
Assert.That(obtained, Is.Null);
}
public void Lex_Property_Colon_String_Diagnostics_DigitAndEndOfFile2()
{
string testCase =
@"""Entier"": {
""type"": ""int"",
""value"": 123456
},
""EntierNegatif"": {
""type"": ""int"",
""value"": -123456
},
""Float"": {
""type"": ""float"",
""value"": 123.3
},
""Float2"": {
""type"": ""float"",
""value"": 123,3
},
""Float3"": {
""type"": ""float"",
""value"": 123.356.789
}";
using (JsonLexer jsonLexer = new JsonLexer(testCase))
{ int
List <SyntaxToken> tokens = jsonLexer.Lex().AsParallel().AsOrdered().ToList();
Assert.IsTrue(!jsonLexer.Diagnostics.Any()); }
}
public void JsonLexerShouldReturnFalse()
{
var input = "false";
var jsonLexer = new JsonLexer();
var tokens = jsonLexer.Lex(input).ToArray();
Assert.AreEqual(1, tokens.Length);
Assert.AreEqual(input, tokens[0].Value);
}
public void Lex_Property_Colon_String_Diagnostics_MissingEndOfString(string testCase)
{
using (JsonLexer jsonLexer = new JsonLexer(testCase))
{
List <SyntaxToken> tokens = jsonLexer.Lex().AsParallel().AsOrdered().ToList();
Assert.IsTrue(jsonLexer.Diagnostics.Any());
Assert.IsTrue(jsonLexer.Diagnostics.All(d => d.Exception is EndOfFileExtractLiteralException));
}
}
public void JsonLexerShouldReturnNull()
{
var input = "true";
var jsonLexer = new JsonLexer();
var tokens = jsonLexer.Lex(input).ToArray();
Assert.AreEqual(1, tokens.Length);
Assert.AreEqual(input, tokens[0].Capture.ToString());
}
public object DeserializeFromJson(ISourceText source, bool caseSensitiveMaps)
{
var lexer = new JsonLexer(source, throwErrors: true);
var parser = new JsonParser(lexer, throwErrors: true, caseSensitiveMaps: caseSensitiveMaps);
parser.Parse();
return(parser.ResultContext.ResultObject);
}
private static object read(ISourceText source, bool caseSensitiveMaps)
{
var lexer = new JsonLexer(source, throwErrors: true);
var parser = new JsonParser(lexer, throwErrors: true, caseSensitiveMaps: caseSensitiveMaps);
parser.Parse();
return(parser.ResultContext.ResultObject);
}
public void JsonLexerShouldReadString()
{
var input = "\"this is a string\"";
var jsonLexer = new JsonLexer();
var tokens = jsonLexer.Lex(input).ToArray();
Assert.AreEqual(1, tokens.Length);
Assert.AreEqual(input, tokens[0].Capture.ToString());
}
public static JsonParser jsonParserFromString(string input)
{
var inputStream = new ANTLRStringStream(input);
var lexer = new JsonLexer(inputStream);
var tokens = new CommonTokenStream(lexer);
var parser = new JsonParser(tokens);
return(parser);
}
public void LexingStringWithTooShortUnicodeEscapeSequenceThrowsAnException()
{
// given
var target = new JsonLexer("\"\\u007\"");
// when, then
var ex = Assert.Throws <JsonLexerException>(() => target.NextToken());
Assert.That(ex.Message, Is.EqualTo("Invalid unicode escape sequence \"\\u007\"\""));
}
public void LexingStringWithNonHexCharacterInUnicodeEscapeSequenceThrowsAnException()
{
// given
var target = new JsonLexer("\"\\u0048\\u0065\\u006C\\u006C\\u006F\\u0020\\uDDGE\\u0021\"");
// when, then
var ex = Assert.Throws <JsonLexerException>(() => target.NextToken());
Assert.That(ex.Message, Is.EqualTo("Invalid unicode escape sequence \"\\uDDG\""));
}
public void LexingStringWithLowSurrogateOnlyThrowsAnException()
{
// given
var target = new JsonLexer("\"\\u0048\\u0065\\u006C\\u006C\\u006F\\u0020\\uDD1E\\u0021\"");
// when, then
var ex = Assert.Throws <JsonLexerException>(() => target.NextToken());
Assert.That(ex.Message, Is.EqualTo("Invalid UTF-16 surrogate pair \"\\uDD1E\""));
}
public void LexingMinusSignWithoutSubsequentDigitsThrowsException()
{
// given
var target = new JsonLexer("-");
// when, then
var ex = Assert.Throws <JsonLexerException>(() => target.NextToken());
Assert.That(ex.Message, Is.EqualTo("Invalid number literal \"-\""));
}
public void LexingUnterminatedStringAfterExcapesequenceThrowsAnException()
{
// given
var target = new JsonLexer("\"foo \\");
// when, then
var ex = Assert.Throws <JsonLexerException>(() => target.NextToken());
Assert.That(ex.Message, Is.EqualTo("Unterminated string literal \"foo \""));
}
public void LexingStringWithInvalidEscapeSequenceThrowsException()
{
// given
var target = new JsonLexer("\"Hello \\a World!\'");
// when, then
var ex = Assert.Throws <JsonLexerException>(() => target.NextToken());
Assert.That(ex.Message, Is.EqualTo("Invalid escape sequence \"\\a\""));
}
public void LexingNumberWithDecimalSeparatorImmediatelyFollowedByUpperCaseExponentThrowsException()
{
// given
var target = new JsonLexer("1.E-5");
// when, then
var ex = Assert.Throws <JsonLexerException>(() => target.NextToken());
Assert.That(ex.Message, Is.EqualTo("Invalid number literal \"1.E-5\""));
}
public void LexingNumberWithExponentAndNoDigitsThrowsException()
{
// given
var target = new JsonLexer("1e");
// when, then
var ex = Assert.Throws <JsonLexerException>(() => target.NextToken());
Assert.That(ex.Message, Is.EqualTo("Invalid number literal \"1e\""));
}
public void LexingNumberWithDecimalSeparatorAndNoSubsequentDigitsThrowsException()
{
// given
var target = new JsonLexer("1234.");
// when, then
var ex = Assert.Throws <JsonLexerException>(() => target.NextToken());
Assert.That(ex.Message, Is.EqualTo("Invalid number literal \"1234.\""));
}
public void LexingNumberWithLeadingZeroThrowsException()
{
// given
var target = new JsonLexer("01234");
// when, then
var ex = Assert.Throws <JsonLexerException>(() => target.NextToken());
Assert.That(ex.Message, Is.EqualTo("Invalid number literal \"01234\""));
}
public void LexingIntegerNumberWithLeadingPlusThrowsException()
{
// given
var target = new JsonLexer("+42");
// when, then
var ex = Assert.Throws <JsonLexerException>(() => target.NextToken());
Assert.That(ex.Message, Is.EqualTo("Unexpected literal \"+42\""));
}
public void LexingStringWithCharactersThatMustBeEscapedButAreNotThrowsAnException()
{
// given
var target = new JsonLexer("\"\n\"");
// when, then
var ex = Assert.Throws <JsonLexerException>(() => target.NextToken());
Assert.That(ex.Message, Is.EqualTo("Invalid control character \"\\u000A\""));
}
public void LexingNumberWithUpperCaseExponentFollowedByMinusAndNoDigitsThrowsException()
{
// given
var target = new JsonLexer("2E-");
// when
var ex = Assert.Throws <JsonLexerException>(() => target.NextToken());
Assert.That(ex.Message, Is.EqualTo("Invalid number literal \"2E-\""));
}
public void LexingNullLiteralWithWrongCasingThrowsAnError()
{
// given
var target = new JsonLexer("nUll");
// when, then
var ex = Assert.Throws <JsonLexerException>(() => target.NextToken());
Assert.That(ex.Message, Is.EqualTo("Unexpected literal \"nUll\""));
}
public void JsonLexerShouldReadSingleDigit()
{
var input = "1";
var jsonLexer = new JsonLexer();
var tokens = jsonLexer.Lex(input).ToArray();
Assert.AreEqual(1, tokens.Length);
Assert.AreEqual(input, tokens[0].Capture.ToString());
Assert.AreEqual(0, tokens[0].Position);
}
public void LexingStringWithWhitespaceOnlyReturnsNullAsNextToken()
{
// given
var target = new JsonLexer(" \r\n\t");
// when
var obtained = target.NextToken();
// then
Assert.That(obtained, Is.Null);
}
public void Lex_Property_Colon_String_Diagnostics_LettersWhiteSpaceAndEndOfFile(string testCase)
{
using (JsonLexer jsonLexer = new JsonLexer(testCase))
{
List <SyntaxToken> tokens = jsonLexer.Lex().AsParallel().AsOrdered().ToList();
Assert.IsTrue(!jsonLexer.Diagnostics.Any());
Assert.AreEqual(tokens[0].Kind, SyntaxKind.Literal);
Assert.AreEqual(tokens[1].Kind, SyntaxKind.Whitespace);
Assert.AreEqual(tokens.Last().Kind, SyntaxKind.EndOfFile);
}
}
public void LexingEmptyStringReturnsNullAsNextToken()
{
// given
var target = new JsonLexer("");
// when
var obtained = target.NextToken();
// then
Assert.That(obtained, Is.Null);
}
/// <summary>
/// Parses the specified JSON input.
/// </summary>
public Node Parse(String input)
{
Assume.NotNull(input, nameof(input));
var stream = new AntlrInputStream(input);
ITokenSource lexer = new JsonLexer(stream);
ITokenStream tokens = new CommonTokenStream(lexer);
var parser = new JsonParser(tokens) {
BuildParseTree = true
};
var tree = parser.start();
var treeFactory = new StandardJsonSyntaxTreeFactory();
var syntaxBuilder = new StandardJsonSyntaxTreeBuilder(treeFactory);
return syntaxBuilder.Visit(input, tree);
}
