private static void RunParseWithCustomLexer(IParseEngine parser)
{
var path = Path.Combine(Directory.GetCurrentDirectory(), "10000.json");
var jsonLexer = new JsonLexer();
using (var stream = File.OpenRead(path))
using (var reader = new StreamReader(stream))
{
var tokens = jsonLexer.Lex(reader);
foreach (var token in tokens)
{
if (token.TokenType != JsonLexer.Whitespace)
{
if (!parser.Pulse(token))
{
Assert.Fail($"unable to parse token {token.TokenType} at {token.Position}");
}
}
}
}
if (!parser.IsAccepted())
{
Assert.Fail("Parse was not accepted");
}
}
public void Lex_Property_Colon_String_Diagnostics_DigitAndEndOfFile2()
{
string testCase =
@"""Entier"": {
""type"": ""int"",
""value"": 123456
},
""EntierNegatif"": {
""type"": ""int"",
""value"": -123456
},
""Float"": {
""type"": ""float"",
""value"": 123.3
},
""Float2"": {
""type"": ""float"",
""value"": 123,3
},
""Float3"": {
""type"": ""float"",
""value"": 123.356.789
}";
using (JsonLexer jsonLexer = new JsonLexer(testCase))
{ int
List <SyntaxToken> tokens = jsonLexer.Lex().AsParallel().AsOrdered().ToList();
Assert.IsTrue(!jsonLexer.Diagnostics.Any()); }
}
public void JsonLexerShouldReturnFalse()
{
var input = "false";
var jsonLexer = new JsonLexer();
var tokens = jsonLexer.Lex(input).ToArray();
Assert.AreEqual(1, tokens.Length);
Assert.AreEqual(input, tokens[0].Value);
}
public void Lex_Property_Colon_String_Diagnostics_MissingEndOfString(string testCase)
{
using (JsonLexer jsonLexer = new JsonLexer(testCase))
{
List <SyntaxToken> tokens = jsonLexer.Lex().AsParallel().AsOrdered().ToList();
Assert.IsTrue(jsonLexer.Diagnostics.Any());
Assert.IsTrue(jsonLexer.Diagnostics.All(d => d.Exception is EndOfFileExtractLiteralException));
}
}
public void JsonLexerShouldReadString()
{
var input = "\"this is a string\"";
var jsonLexer = new JsonLexer();
var tokens = jsonLexer.Lex(input).ToArray();
Assert.AreEqual(1, tokens.Length);
Assert.AreEqual(input, tokens[0].Capture.ToString());
}
public void JsonLexerShouldReturnNull()
{
var input = "true";
var jsonLexer = new JsonLexer();
var tokens = jsonLexer.Lex(input).ToArray();
Assert.AreEqual(1, tokens.Length);
Assert.AreEqual(input, tokens[0].Capture.ToString());
}
public void JsonLexerShouldReadSingleDigit()
{
var input = "1";
var jsonLexer = new JsonLexer();
var tokens = jsonLexer.Lex(input).ToArray();
Assert.AreEqual(1, tokens.Length);
Assert.AreEqual(input, tokens[0].Capture.ToString());
Assert.AreEqual(0, tokens[0].Position);
}
public void Lex_Property_Colon_String_Diagnostics_LettersWhiteSpaceAndEndOfFile(string testCase)
{
using (JsonLexer jsonLexer = new JsonLexer(testCase))
{
List <SyntaxToken> tokens = jsonLexer.Lex().AsParallel().AsOrdered().ToList();
Assert.IsTrue(!jsonLexer.Diagnostics.Any());
Assert.AreEqual(tokens[0].Kind, SyntaxKind.Literal);
Assert.AreEqual(tokens[1].Kind, SyntaxKind.Whitespace);
Assert.AreEqual(tokens.Last().Kind, SyntaxKind.EndOfFile);
}
}
public void LexerTestListOfObjects()
{
string[] lex = JsonLexer.Lex("[{\"key1\":\"val1\"},{\"key2\":\"val2\"}]");
Assert.IsNotNull(lex);
string[] expected = new string[] { "[", "{", "key1", ":", "val1", "}", ",", "{", "key2", ":", "val2", "}", "]" };
for (int i = 0; i < expected.Length; i++)
{
Assert.That(lex[i] == expected[i]);
}
}
public void LexerTestSimpleList()
{
string[] lex = JsonLexer.Lex("[\"li1\",\"li2\"]");
Assert.IsNotNull(lex);
string[] expected = new string[] { "[", "li1", ",", "li2", "]" };
for (int i = 0; i < expected.Length; i++)
{
Assert.That(lex[i] == expected[i]);
}
}
public void LexerTestObjectWithListValue()
{
string[] lex = JsonLexer.Lex("{\"key1\":[\"li1\",\"li2\"]}");
Assert.IsNotNull(lex);
string[] expected = new string[] { "{", "key1", ":", "[", "li1", ",", "li2", "]", "}" };
for (int i = 0; i < expected.Length; i++)
{
Assert.That(lex[i] == expected[i]);
}
}
public void LexerTestSimpleObject()
{
string[] lex = JsonLexer.Lex("{\"key\":\"value\"}");
Assert.IsNotNull(lex);
string[] expected = new string[] { "{", "key", ":", "value", "}" };
for (int i = 0; i < expected.Length; i++)
{
Assert.That(lex[i] == expected[i]);
}
}
public void Lex_Property_Colon_String()
{
using (JsonLexer jsonLexer = new JsonLexer("\"pattern\": \"^\\\\{[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}\\}$\""))
{
List <SyntaxToken> tokens = jsonLexer.Lex().AsParallel().AsOrdered().ToList();
Assert.AreEqual("pattern", tokens[0].Text);
Assert.AreEqual(":", tokens[1].Text);
Assert.AreEqual(" ", tokens[2].Text);
Assert.AreEqual("^\\\\{[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}\\}$", tokens[3].Text);
Assert.AreEqual(tokens[4].Kind, SyntaxKind.EndOfFile);
}
}
public void LexerTestOpenAiExample()
{
string[] lex = JsonLexer.Lex("{\"id\": \"cmpl - uqkvlQyYK7bGYrRHQ0eXlWi7\",\"object\": \"text_completion\", \"created\": 1589478378,\"model\": \"davinci:2020-05-03\",\"choices\": [{\"text\": \" there was a girl who\",\"index\": 0,\"logprobs\": null,\"finish_reason\": \"length\"}]} ");
Assert.IsNotNull(lex);
string[] expected = new string[] { "{", "id", ":", "cmpl - uqkvlQyYK7bGYrRHQ0eXlWi7", ",", "object", ":", "text_completion", ",", "created", ":", "1589478378", ",", "model", ":", "davinci:2020-05-03", ",", "choices", ":", "[", "{", "text", ":", " there was a girl who", ",", "index", ":", "0", ",", "logprobs", ":", "null", ",", "finish_reason", ":", "length", "}", "]", "}" };
for (int i = 0; i < expected.Length; i++)
{
Assert.That(lex[i] == expected[i]);
}
}
public void Lex_Property_Colon_String_Diagnostics_WhiteSpaceAndEndOfFile()
{
string testCase = new String(new[] { ' ', ' ', ' ', ' ', ' ' });
using (JsonLexer jsonLexer = new JsonLexer(testCase))
{
List <SyntaxToken> tokens = jsonLexer.Lex().AsParallel().AsOrdered().ToList();
Assert.IsTrue(!jsonLexer.Diagnostics.Any());
Assert.AreEqual(tokens.Count, 2);
Assert.AreEqual(tokens[0].Kind, SyntaxKind.Whitespace);
Assert.AreEqual(tokens[0].Text.Length, 5);
Assert.AreEqual(tokens[1].Kind, SyntaxKind.EndOfFile);
}
}
public void LexFirstCurlyBracket()
{
List <SyntaxToken> tokens = new List <SyntaxToken>();
Stopwatch watch = new Stopwatch();
watch.Start();
using (JsonLexer jsonLexer = new JsonLexer(new FileInfo(@"C:\Program Files (x86)\Audiokinetic\Wwise 2018.1.6.6858\Authoring\Data\Schemas\WwiseAuthoringAPI.json")))
{
tokens = jsonLexer.Lex().AsParallel().AsOrdered().ToList();
Assert.IsTrue(!jsonLexer.Diagnostics.Any());
watch.Stop();
TimeSpan elapsed = watch.Elapsed;
Assert.IsTrue(elapsed < TimeSpan.FromMilliseconds(250));
}
}
public void JsonLexerShouldReturnObjectTokens()
{
var input = "{\"name\":\"something\",\"id\":12345}";
var jsonLexer = new JsonLexer();
var tokens = jsonLexer.Lex(input).ToArray();
Assert.AreEqual(9, tokens.Length);
Assert.AreEqual(JsonLexer.OpenBrace, tokens[0].TokenType);
Assert.AreEqual(JsonLexer.String, tokens[1].TokenType);
Assert.AreEqual(JsonLexer.Colon, tokens[2].TokenType);
Assert.AreEqual(JsonLexer.String, tokens[3].TokenType);
Assert.AreEqual(JsonLexer.Comma, tokens[4].TokenType);
Assert.AreEqual(JsonLexer.String, tokens[5].TokenType);
Assert.AreEqual(JsonLexer.Colon, tokens[6].TokenType);
Assert.AreEqual(JsonLexer.Number, tokens[7].TokenType);
Assert.AreEqual(JsonLexer.CloseBrace, tokens[8].TokenType);
}
public void Lex_Property_Colon_String_Diagnostics_WhiteSpaceAndEndOfFile2()
{
string testCase =
@"""propertyName"": {
""type"": ""string"",
""pattern"": ""^[a-zA-Z0-9 _]+$""
},
""objectPathArg"": {
""type"": ""string"",
""pattern"": ""^\\\\""
}";
using (JsonLexer jsonLexer = new JsonLexer(testCase))
{
List <SyntaxToken> tokens = jsonLexer.Lex().AsParallel().AsOrdered().ToList();
Assert.IsTrue(!jsonLexer.Diagnostics.Any());
}
}
public void JsonLexerShouldReturnArrayTokens()
{
var input = "[1,2,3]";
var jsonLexer = new JsonLexer();
var tokens = jsonLexer.Lex(input).ToArray();
Assert.AreEqual(7, tokens.Length);
for (var i = 0; i < input.Length; i++)
{
Assert.AreEqual(input[i], tokens[i].Capture[0]);
}
Assert.AreEqual(JsonLexer.OpenBracket, tokens[0].TokenType);
Assert.AreEqual(JsonLexer.Number, tokens[1].TokenType);
Assert.AreEqual(JsonLexer.Comma, tokens[2].TokenType);
Assert.AreEqual(JsonLexer.Number, tokens[3].TokenType);
Assert.AreEqual(JsonLexer.Comma, tokens[4].TokenType);
Assert.AreEqual(JsonLexer.Number, tokens[5].TokenType);
Assert.AreEqual(JsonLexer.CloseBracket, tokens[6].TokenType);
}
private static void RunParseWithCustomLexer(IParseEngine parser)
{
var path = Path.Combine(Directory.GetCurrentDirectory(), "10000.json");
var jsonLexer = new JsonLexer();
var tokens = jsonLexer.Lex(File.ReadAllText(path));
foreach (var token in tokens)
{
if (!Equals(token.TokenName, JsonLexer.Whitespace))
{
if (!parser.Pulse(token))
{
Assert.Fail($"unable to parse token {token.TokenName} at {token.Position}");
}
}
}
if (!parser.IsAccepted())
{
Assert.Fail("Parse was not accepted");
}
}
