protected ParserRuleContext ParseTokens(SourceCodeFile sourceCodeFile, AntlrMemoryErrorListener errorListener, BufferedTokenStream codeTokenStream, Func <ITokenStream, Parser> initParserFunc = null, Func <Parser, ParserRuleContext> parseFunc = null) { Parser parser = initParserFunc != null?initParserFunc(codeTokenStream) : InitParser(codeTokenStream); parser.RemoveErrorListeners(); Parser = parser; ParserRuleContext syntaxTree; if (UseFastParseStrategyAtFirst) { parser.Interpreter.PredictionMode = PredictionMode.Sll; parser.ErrorHandler = new BailErrorStrategy(); parser.TrimParseTree = true; parserLock.EnterReadLock(); try { syntaxTree = parseFunc != null?parseFunc(parser) : Parse(parser); } catch (ParseCanceledException) { parserLock.ExitReadLock(); parser.AddErrorListener(errorListener); codeTokenStream.Reset(); parser.Reset(); parser.Interpreter.PredictionMode = PredictionMode.Ll; parser.ErrorHandler = new DefaultErrorStrategy(); parserLock.EnterReadLock(); syntaxTree = parseFunc != null?parseFunc(parser) : Parse(parser); } finally { parserLock.ExitReadLock(); } } else { parser.AddErrorListener(errorListener); parserLock.EnterReadLock(); try { syntaxTree = parseFunc != null?parseFunc(parser) : Parse(parser); } finally { parserLock.ExitReadLock(); } } ClearParserCacheIfRequired(parser); #if DEBUG var tree = syntaxTree.ToStringTree(parser); #endif return(syntaxTree); }
public SqlSyntaxErrorListener AddParserErrorListener(Parser parser) { SqlSyntaxErrorListener errorListener = new SqlSyntaxErrorListener(); parser.AddErrorListener(errorListener); return(errorListener); }
protected ParserRuleContext ParseTokens(CodeFile sourceCodeFile, AntlrMemoryErrorListener errorListener, BufferedTokenStream codeTokenStream, Func <ITokenStream, Parser> initParserFunc = null, Func <Parser, ParserRuleContext> parseFunc = null) { Parser parser = initParserFunc != null?initParserFunc(codeTokenStream) : InitParser(codeTokenStream); parser.Interpreter = new ParserATNSimulator(parser, GetOrCreateAtn(false)); parser.RemoveErrorListeners(); Parser = parser; ParserRuleContext syntaxTree = null; if (UseFastParseStrategyAtFirst) { parser.Interpreter.PredictionMode = PredictionMode.Sll; parser.ErrorHandler = new BailErrorStrategy(); parser.TrimParseTree = true; try { syntaxTree = parseFunc != null?parseFunc(parser) : Parse(parser); } catch (ParseCanceledException) { parser.AddErrorListener(errorListener); codeTokenStream.Reset(); parser.Reset(); parser.Interpreter.PredictionMode = PredictionMode.Ll; parser.ErrorHandler = new DefaultErrorStrategy(); syntaxTree = parseFunc != null?parseFunc(parser) : Parse(parser); } } else { parser.AddErrorListener(errorListener); syntaxTree = parseFunc != null?parseFunc(parser) : Parse(parser); } return(syntaxTree); }
public ParserRuleContext Parse(LexicalExtension <SyntaxToken> extension, Scope scope) { var text = RoslynCompiler.TokensToString(extension.Body); //td: token matching AntlrInputStream stream = new AntlrInputStream(text); ITokenSource lexer = GetLexer(stream); ITokenStream tokenStream = new CommonTokenStream(lexer); Parser parser = GetParser(tokenStream); parser.AddErrorListener(new AntlrErrors <IToken>(scope, extension.BodyStart)); var result = GetRoot(parser); if (parser.NumberOfSyntaxErrors > 0) { return(null); } return(result); }
public int parseFile(ICharStream input, int thread) { Checksum checksum = new CRC32(); Debug.Assert(thread >= 0 && thread < NUMBER_OF_THREADS); try { IParseTreeListener listener = sharedListeners[thread]; if (listener == null) { listener = (IParseTreeListener)Activator.CreateInstance(listenerClass); sharedListeners[thread] = listener; } Lexer lexer = sharedLexers[thread]; if (REUSE_LEXER && lexer != null) { lexer.SetInputStream(input); } else { lexer = (Lexer)lexerCtor.Invoke(new object[] { input }); sharedLexers[thread] = lexer; if (!ENABLE_LEXER_DFA) { lexer.Interpreter = new NonCachingLexerATNSimulator(lexer, lexer.Atn); } else if (!REUSE_LEXER_DFA) { lexer.Interpreter = new LexerATNSimulator(lexer, sharedLexerATNs[thread]); } } lexer.Interpreter.optimize_tail_calls = OPTIMIZE_TAIL_CALLS; if (ENABLE_LEXER_DFA && !REUSE_LEXER_DFA) { lexer.Interpreter.atn.ClearDFA(); } CommonTokenStream tokens = new CommonTokenStream(lexer); tokens.Fill(); Interlocked.Add(ref tokenCount, tokens.Size); if (COMPUTE_CHECKSUM) { foreach (IToken token in tokens.GetTokens()) { updateChecksum(checksum, token); } } if (!RUN_PARSER) { return((int)checksum.Value); } Parser parser = sharedParsers[thread]; if (REUSE_PARSER && parser != null) { parser.SetInputStream(tokens); } else { Parser newParser = (Parser)parserCtor.Invoke(new object[] { tokens }); parser = newParser; sharedParsers[thread] = parser; } parser.RemoveErrorListeners(); if (!TWO_STAGE_PARSING) { parser.AddErrorListener(DescriptiveErrorListener.INSTANCE); parser.AddErrorListener(new SummarizingDiagnosticErrorListener()); } if (!ENABLE_PARSER_DFA) { parser.Interpreter = new NonCachingParserATNSimulator(parser, parser.Atn); } else if (!REUSE_PARSER_DFA) { parser.Interpreter = new ParserATNSimulator(parser, sharedParserATNs[thread]); } if (ENABLE_PARSER_DFA && !REUSE_PARSER_DFA) { parser.Interpreter.atn.ClearDFA(); } parser.Interpreter.PredictionMode = TWO_STAGE_PARSING ? PredictionMode.Sll : PREDICTION_MODE; parser.Interpreter.force_global_context = FORCE_GLOBAL_CONTEXT && !TWO_STAGE_PARSING; parser.Interpreter.always_try_local_context = TRY_LOCAL_CONTEXT_FIRST || TWO_STAGE_PARSING; parser.Interpreter.optimize_ll1 = OPTIMIZE_LL1; parser.Interpreter.optimize_unique_closure = OPTIMIZE_UNIQUE_CLOSURE; parser.Interpreter.optimize_tail_calls = OPTIMIZE_TAIL_CALLS; parser.Interpreter.tail_call_preserves_sll = TAIL_CALL_PRESERVES_SLL; parser.Interpreter.treat_sllk1_conflict_as_ambiguity = TREAT_SLLK1_CONFLICT_AS_AMBIGUITY; parser.BuildParseTree = BUILD_PARSE_TREES; if (!BUILD_PARSE_TREES && BLANK_LISTENER) { parser.AddParseListener(listener); } if (BAIL_ON_ERROR || TWO_STAGE_PARSING) { parser.ErrorHandler = new BailErrorStrategy(); } MethodInfo parseMethod = parserClass.GetMethod(entryPoint); object parseResult; IParseTreeListener checksumParserListener = null; try { if (COMPUTE_CHECKSUM) { checksumParserListener = new ChecksumParseTreeListener(checksum); parser.AddParseListener(checksumParserListener); } parseResult = parseMethod.Invoke(parser, null); } catch (TargetInvocationException ex) { if (!TWO_STAGE_PARSING) { throw; } string sourceName = tokens.SourceName; sourceName = !string.IsNullOrEmpty(sourceName) ? sourceName + ": " : ""; Console.Error.WriteLine(sourceName + "Forced to retry with full context."); if (!(ex.InnerException is ParseCanceledException)) { throw; } tokens.Reset(); if (REUSE_PARSER && sharedParsers[thread] != null) { parser.SetInputStream(tokens); } else { Parser newParser = (Parser)parserCtor.Invoke(new object[] { tokens }); parser = newParser; sharedParsers[thread] = parser; } parser.RemoveErrorListeners(); parser.AddErrorListener(DescriptiveErrorListener.INSTANCE); parser.AddErrorListener(new SummarizingDiagnosticErrorListener()); if (!ENABLE_PARSER_DFA) { parser.Interpreter = new NonCachingParserATNSimulator(parser, parser.Atn); } parser.Interpreter.PredictionMode = PREDICTION_MODE; parser.Interpreter.force_global_context = FORCE_GLOBAL_CONTEXT; parser.Interpreter.always_try_local_context = TRY_LOCAL_CONTEXT_FIRST; parser.Interpreter.optimize_ll1 = OPTIMIZE_LL1; parser.Interpreter.optimize_unique_closure = OPTIMIZE_UNIQUE_CLOSURE; parser.Interpreter.optimize_tail_calls = OPTIMIZE_TAIL_CALLS; parser.Interpreter.tail_call_preserves_sll = TAIL_CALL_PRESERVES_SLL; parser.Interpreter.treat_sllk1_conflict_as_ambiguity = TREAT_SLLK1_CONFLICT_AS_AMBIGUITY; parser.BuildParseTree = BUILD_PARSE_TREES; if (!BUILD_PARSE_TREES && BLANK_LISTENER) { parser.AddParseListener(listener); } if (BAIL_ON_ERROR) { parser.ErrorHandler = new BailErrorStrategy(); } parseResult = parseMethod.Invoke(parser, null); } finally { if (checksumParserListener != null) { parser.RemoveParseListener(checksumParserListener); } } Assert.IsInstanceOfType(parseResult, typeof(IParseTree)); if (BUILD_PARSE_TREES && BLANK_LISTENER) { ParseTreeWalker.Default.Walk(listener, (ParserRuleContext)parseResult); } } catch (Exception e) { if (!REPORT_SYNTAX_ERRORS && e is ParseCanceledException) { return((int)checksum.Value); } throw; } return((int)checksum.Value); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.MemberAccessException"/> /// <exception cref="System.Reflection.TargetInvocationException"/> /// <exception cref="Javax.Print.PrintException"/> protected internal virtual void Process <_T0>(Lexer lexer, Type <_T0> parserClass, Parser parser, Stream @is, TextReader r) where _T0 : Parser { try { AntlrInputStream input = new AntlrInputStream(r); lexer.SetInputStream(input); CommonTokenStream tokens = new CommonTokenStream(lexer); tokens.Fill(); if (showTokens) { foreach (object tok in tokens.GetTokens()) { System.Console.Out.WriteLine(tok); } } if (startRuleName.Equals(LexerStartRuleName)) { return; } if (diagnostics) { parser.AddErrorListener(new DiagnosticErrorListener()); parser.Interpreter.PredictionMode = PredictionMode.LlExactAmbigDetection; } if (printTree || gui || psFile != null) { parser.BuildParseTree = true; } if (Sll) { // overrides diagnostics parser.Interpreter.PredictionMode = PredictionMode.Sll; } parser.SetInputStream(tokens); parser.Trace = trace; try { MethodInfo startRule = parserClass.GetMethod(startRuleName, (Type[])null); ParserRuleContext tree = (ParserRuleContext)startRule.Invoke(parser, (object[])null); if (printTree) { System.Console.Out.WriteLine(tree.ToStringTree(parser)); } if (gui) { tree.Inspect(parser); } if (psFile != null) { tree.Save(parser, psFile); } } catch (NoSuchMethodException) { // Generate postscript System.Console.Error.WriteLine("No method for rule " + startRuleName + " or it has arguments"); } } finally { if (r != null) { r.Close(); } if (@is != null) { @is.Close(); } } }