private static void testJSONLexer()
{
var builder = new ParserBuilder <JsonToken, JSon>();
var parser = builder.BuildParser(new JSONParser(), ParserType.EBNF_LL_RECURSIVE_DESCENT, "root");
var source = "{ \"k\" : 1;\"k2\" : 1.1;\"k3\" : null;\"k4\" : false}";
//source = File.ReadAllText("test.json");
var lexer = new JSONLexer();
var sw = new Stopwatch();
sw.Start();
var lexresult = lexer.Tokenize(source);
if (lexresult.IsOk)
{
var tokens = lexresult.Tokens;
sw.Stop();
Console.WriteLine($"hard coded lexer {tokens.Count()} tokens in {sw.ElapsedMilliseconds}ms");
var sw2 = new Stopwatch();
var start = DateTime.Now.Millisecond;
sw2.Start();
lexresult = parser.Result.Lexer.Tokenize(source);
if (lexresult.IsOk)
{
tokens = lexresult.Tokens;
sw2.Stop();
var end = DateTime.Now.Millisecond;
Console.WriteLine(
$"old lexer {tokens.Count()} tokens in {sw2.ElapsedMilliseconds}ms / {end - start}ms");
}
}
}
/// <summary>
/// Checks all pattern match attributes against specified member info until first match found
/// </summary>
public static bool Check(MemberInfo info, JSONLexer content)
{
var attrs = info.GetCustomAttributes(typeof(JSONPatternMatchAttribute), true).Cast<JSONPatternMatchAttribute>();
foreach(var attr in attrs)
if (attr.Match(content)) return true;
return false;
}
private static object read(ISourceText source)
{
var lexer = new JSONLexer(source, throwErrors: true);
var parser = new JSONParser(lexer, throwErrors: true);
parser.Parse();
return(parser.ResultContext.ResultObject);
}
public DataItem FromFile(Stream dataFile)
{
ICharStream chars = CharStreams.fromStream(dataFile);
JSONLexer jsonLexer = new JSONLexer(chars);
CommonTokenStream commonTokenStream = new CommonTokenStream(jsonLexer);
JSONParser jsonParser = new JSONParser(commonTokenStream);
JSONConverterVisitor visitor = new JSONConverterVisitor();
return(visitor.VisitJson(jsonParser.json()));
}
private static IJsonable parse(ICharStream ins)
{
JSONLexer lexer = new JSONLexer(ins);
CommonTokenStream tokens = new CommonTokenStream(lexer);
JSONParser parser = new JSONParser(tokens);
JsonContext ctx = parser.json();
ParseTreeWalker walker = new ParseTreeWalker();
JSONAnsonListener lstner = new JSONAnsonListener();
walker.Walk(lstner, ctx);
return(lstner.ParsedEnvelope());
}
private void ParseJson(string input)
{
AntlrInputStream stream = new AntlrInputStream(input);
ITokenSource lexer = new JSONLexer(stream);
ITokenStream tokens = new CommonTokenStream(lexer);
JSONParser parser = new JSONParser(tokens);
parser.BuildParseTree = true;
JSONParser.JsonContext tree = parser.json();
if (tree != null)
{
var builder = new TreeBuilder(parser, tree, treeModel);
builder.Build();
}
}
public ParserRuleContext parse(IEnumerable <SyntaxToken> tokens, Scope scope, int offset)
{
var text = RoslynCompiler.TokensToString(tokens);
AntlrInputStream stream = new AntlrInputStream(text);
ITokenSource lexer = new JSONLexer(stream);
ITokenStream tokenStream = new CommonTokenStream(lexer);
JSONParser parser = new JSONParser(tokenStream);
parser.AddErrorListener(new AntlrErrors <IToken>(scope, offset));
var result = parser.json();
if (parser.NumberOfSyntaxErrors > 0)
{
return(null);
}
return(result);
}
public JSONToken(JSONLexer lexer, JSONTokenType type, SourcePosition startPos, SourcePosition endPos, string text, object value = null)
: base(lexer, startPos, endPos, text, value)
{
Type = type;
}
/// <summary> /// Override to perform actual pattern matching, i.e. the one that uses FSM /// </summary> public abstract bool Match(JSONLexer content);
