/// <summary> /// For repeated use of a tree pattern, compile it to a /// <see cref="ParseTreePattern"/> /// using this method. /// </summary> public virtual ParseTreePattern Compile(string pattern, int patternRuleIndex) { IList <IToken> tokenList = Tokenize(pattern); ListTokenSource tokenSrc = new ListTokenSource(tokenList); CommonTokenStream tokens = new CommonTokenStream(tokenSrc); ParserInterpreter parserInterp = new ParserInterpreter(parser.GrammarFileName, parser.Vocabulary, Arrays.AsList(parser.RuleNames), parser.GetATNWithBypassAlts(), tokens); IParseTree tree = null; try { parserInterp.ErrorHandler = new BailErrorStrategy(); tree = parserInterp.Parse(patternRuleIndex); } catch (ParseCanceledException e) { // System.out.println("pattern tree = "+tree.toStringTree(parserInterp)); throw (RecognitionException)e.InnerException; } catch (RecognitionException) { throw; } catch (Exception e) { throw new ParseTreePatternMatcher.CannotInvokeStartRule(e); } // Make sure tree pattern compilation checks for a complete parse if (tokens.LA(1) != TokenConstants.EOF) { throw new ParseTreePatternMatcher.StartRuleDoesNotConsumeFullPattern(); } return(new ParseTreePattern(this, pattern, patternRuleIndex, tree)); }
static void Main(string[] args) { try { string fileName = "../../../../Text"; if (args.Length > 0) { fileName = args[0]; } var code = File.ReadAllText(fileName); var codeStream = new AntlrInputStream(code); var lexer = new __TemplateGrammarName__Lexer(codeStream); lexer.RemoveErrorListeners(); lexer.AddErrorListener(new LexerErrorListener()); var stopwatch = Stopwatch.StartNew(); var tokens = lexer.GetAllTokens(); stopwatch.Stop(); Console.WriteLine("LexerTime {0}", stopwatch.Elapsed); Console.WriteLine("Tokens {0}", tokens.TokensToString()); /*$ParserPart*/ string rootRule = null; bool notParse = false; if (args.Length > 1) { rootRule = args[1]; if (args.Length > 2) { bool.TryParse(args[2], out notParse); } } if (!notParse) { var tokensSource = new ListTokenSource(tokens); var tokensStream = new CommonTokenStream(tokensSource); var parser = new __TemplateGrammarName__Parser(tokensStream); parser.RemoveErrorListeners(); parser.AddErrorListener(new ParserErrorListener()); stopwatch.Restart(); string ruleName = rootRule == null ? __TemplateGrammarName__Parser.ruleNames[0] : rootRule; var rootMethod = typeof(__TemplateGrammarName__Parser).GetMethod(ruleName); var ast = (ParserRuleContext)rootMethod.Invoke(parser, new object[0]); stopwatch.Stop(); Console.WriteLine("ParserTime {0}", stopwatch.Elapsed); Console.WriteLine("Tree {0}", ast.ToStringTree(parser)); } /*ParserPart$*/ } catch (Exception ex) { Console.Error.WriteLine(ex.ToString().Replace("\r", "").Replace("\n", "")); } }
private CSharpParser.Compilation_unitContext ParseFile(string file) { IList <IToken> codeTokens = new List <IToken>(); string sourceCode = File.ReadAllText(file); Lexer preprocessorLexer = new CSharpLexer(new AntlrInputStream(sourceCode)); IList <IToken> tokens = preprocessorLexer.GetAllTokens(); IList <IToken> directiveTokens = new List <IToken>(); ListTokenSource directiveTokenSource = new ListTokenSource(directiveTokens); CommonTokenStream directiveTokenStream = new CommonTokenStream(directiveTokenSource, CSharpLexer.DIRECTIVE); CSharpPreprocessorParser preprocessorParser = new CSharpPreprocessorParser(directiveTokenStream); int index = 0; bool compiliedTokens = true; while (index < tokens.Count) { IToken token = tokens[index]; if (token.Type == CSharpLexer.SHARP) { directiveTokens.Clear(); int directiveTokenIndex = index + 1; while (directiveTokenIndex < tokens.Count && tokens[directiveTokenIndex].Type != CSharpLexer.Eof && tokens[directiveTokenIndex].Type != CSharpLexer.DIRECTIVE_NEW_LINE && tokens[directiveTokenIndex].Type != CSharpLexer.SHARP) { if (tokens[directiveTokenIndex].Channel != Lexer.Hidden && tokens[directiveTokenIndex].Channel != CSharpLexer.COMMENTS_CHANNEL) { directiveTokens.Add(tokens[directiveTokenIndex]); } directiveTokenIndex++; } directiveTokenSource = new ListTokenSource(directiveTokens); directiveTokenStream = new CommonTokenStream(directiveTokenSource, CSharpLexer.DIRECTIVE); preprocessorParser.TokenStream = directiveTokenStream; CSharpPreprocessorParser.Preprocessor_directiveContext directive = preprocessorParser.preprocessor_directive(); // if true than next code is valid and not ignored. compiliedTokens = directive.value; index = directiveTokenIndex - 1; } else if (token.Channel != Lexer.Hidden && token.Type != CSharpLexer.DIRECTIVE_NEW_LINE && token.Channel != CSharpLexer.COMMENTS_CHANNEL && compiliedTokens) { codeTokens.Add(token); // Collect code tokens. } index++; } ListTokenSource codeTokenSource = new ListTokenSource(codeTokens); CommonTokenStream codeTokenStream = new CommonTokenStream(codeTokenSource); CSharpParser parser = new CSharpParser(codeTokenStream); return(parser.compilation_unit()); }
public ParseTree Parse(IList <IToken> tokens, out TimeSpan parserTimeSpan) { if (SourceFile == null) { throw new ArgumentNullException(nameof(SourceFile)); } if (ErrorListener == null) { ErrorListener = new AntlrMemoryErrorListener(); ErrorListener.Logger = Logger; ErrorListener.LineOffset = LineOffset; } ErrorListener.SourceFile = SourceFile; AntlrParseTree result = null; try { var commentTokens = new List <IToken>(); foreach (IToken token in tokens) { if (token.Channel == CommentsChannel) { commentTokens.Add(token); } } var stopwatch = Stopwatch.StartNew(); var codeTokenSource = new ListTokenSource(tokens); var codeTokenStream = new CommonTokenStream(codeTokenSource); ParserRuleContext syntaxTree = ParseTokens(ErrorListener, codeTokenStream); stopwatch.Stop(); parserTimeSpan = stopwatch.Elapsed; result = Create(syntaxTree); result.Tokens = tokens; result.Comments = commentTokens; result.SourceFile = SourceFile; } catch (Exception ex) when(!(ex is ThreadAbortException)) { Logger.LogError(new ParsingException(SourceFile, ex)); } finally { HandleMemoryConsumption(); } return(result); }
public ITokenSource Edit(ITokenSource tokens) { CommonTokenStream s = new CommonTokenStream(tokens); HtmlParser parser = new HtmlParser(s); s.Fill(); var program = parser.htmlDocument(); visitor.Visit(program, s.GetTokens()); var list = visitor.GetResult(); var result = new ListTokenSource(list); result.TokenFactory = tokens.TokenFactory; return(result); }
public virtual ITokenSource Edit(ITokenSource tokens) { CommonTokenStream s = new CommonTokenStream(tokens); JsParser parser = new JsParser(s); s.Fill(); var program = parser.program(); visitor.Visit(program, s.GetTokens()); var source = new ListTokenSource(visitor.GetResult()); source.TokenFactory = tokens.TokenFactory; return(source); }
public ITokenSource Edit(ITokenSource tokens) { CommonTokenStream s = new CommonTokenStream(tokens); JsParser parser = new JsParser(s); s.Fill(); IVisitorTree v = new JsVisitorChangeLiteralString(factoryNames, tokens.TokenFactory); var program = parser.program(); v.Visit(program, s.GetTokens()); var source = new ListTokenSource(v.GetResult()); source.TokenFactory = tokens.TokenFactory; return(source); }
private void ParseCSharp(string input) { Lexer preprocessorLexer = new CSharpLexer(new AntlrInputStream(input)); // Collect all tokens with lexer (CSharpLexer.g4). var tokens = preprocessorLexer.GetAllTokens(); //TODO: handle preprocessor tokens var codeTokenSource = new ListTokenSource(tokens); var codeTokenStream = new CommonTokenStream(codeTokenSource); CSharpParser parser = new CSharpParser(codeTokenStream); // Parse syntax tree (CSharpParser.g4) var tree = parser.compilation_unit(); if (tree != null) { var builder = new TreeBuilder(parser, tree, treeModel); builder.Build(); } }
public ParseTree Parse(CodeFile sourceCodeFile) { AntlrParseTree result = null; var filePath = sourceCodeFile.RelativeName; if (sourceCodeFile.Code != null) { var errorListener = new AntlrMemoryErrorListener(); errorListener.CodeFile = sourceCodeFile; errorListener.Logger = Logger; errorListener.LineOffset = LineOffset; try { var preprocessedText = PreprocessText(sourceCodeFile); AntlrInputStream inputStream; if (Language.IsCaseInsensitive) { inputStream = new AntlrCaseInsensitiveInputStream(preprocessedText, CaseInsensitiveType); } else { inputStream = new AntlrInputStream(preprocessedText); } inputStream.name = filePath; Lexer lexer = InitLexer(inputStream); lexer.Interpreter = new LexerATNSimulator(lexer, GetOrCreateAtn(true)); lexer.RemoveErrorListeners(); lexer.AddErrorListener(errorListener); var commentTokens = new List <IToken>(); var stopwatch = Stopwatch.StartNew(); IList <IToken> tokens = lexer.GetAllTokens(); stopwatch.Stop(); long lexerTimeSpanTicks = stopwatch.ElapsedTicks; foreach (IToken token in tokens) { if (token.Channel == CommentsChannel) { commentTokens.Add(token); } } stopwatch.Restart(); var codeTokenSource = new ListTokenSource(tokens); var codeTokenStream = new CommonTokenStream(codeTokenSource); ParserRuleContext syntaxTree = ParseTokens(sourceCodeFile, errorListener, codeTokenStream); stopwatch.Stop(); long parserTimeSpanTicks = stopwatch.ElapsedTicks; result = Create(syntaxTree); result.LexerTimeSpan = new TimeSpan(lexerTimeSpanTicks); result.ParserTimeSpan = new TimeSpan(parserTimeSpanTicks); result.Tokens = tokens; result.Comments = commentTokens; } catch (Exception ex) when(!(ex is ThreadAbortException)) { Logger.LogError(new ParsingException(sourceCodeFile, ex)); if (result == null) { result = Create(null); } } finally { Interlocked.Increment(ref processedFilesCount); Interlocked.Add(ref processedBytesCount, sourceCodeFile.Code.Length); } } else { result = Create(null); } result.SourceCodeFile = sourceCodeFile; return(result); }
protected virtual ParseTree TokenizeAndParse(SourceCodeFile sourceCodeFile) { AntlrParseTree result = null; var filePath = Path.Combine(sourceCodeFile.RelativePath, sourceCodeFile.Name); if (sourceCodeFile.Code != null) { var errorListener = new AntlrMemoryErrorListener(); errorListener.FileName = filePath; errorListener.FileData = sourceCodeFile.Code; errorListener.Logger = Logger; errorListener.LineOffset = sourceCodeFile.LineOffset; try { var preprocessedText = PreprocessText(sourceCodeFile); AntlrInputStream inputStream; if (Language.IsCaseInsensitive()) { inputStream = new AntlrCaseInsensitiveInputStream(preprocessedText, CaseInsensitiveType); } else { inputStream = new AntlrInputStream(preprocessedText); } inputStream.name = filePath; Lexer lexer = InitLexer(inputStream); Lexer = lexer; lexer.RemoveErrorListeners(); lexer.AddErrorListener(errorListener); var commentTokens = new List <IToken>(); var stopwatch = Stopwatch.StartNew(); IList <IToken> tokens = GetAllTokens(lexer); stopwatch.Stop(); long lexerTimeSpanTicks = stopwatch.ElapsedTicks; #if DEBUG var codeTokensStr = AntlrHelper.GetTokensString(tokens, Vocabulary, onlyDefaultChannel: false); #endif ClearLexerCacheIfRequired(lexer); foreach (var token in tokens) { if (token.Channel == CommentsChannel) { commentTokens.Add(token); } } stopwatch.Restart(); var codeTokenSource = new ListTokenSource(tokens); var codeTokenStream = new CommonTokenStream(codeTokenSource); ParserRuleContext syntaxTree = ParseTokens(sourceCodeFile, errorListener, codeTokenStream); stopwatch.Stop(); long parserTimeSpanTicks = stopwatch.ElapsedTicks; IncrementProcessedFilesCount(); result = Create(syntaxTree); result.LexerTimeSpan = new TimeSpan(lexerTimeSpanTicks); result.ParserTimeSpan = new TimeSpan(parserTimeSpanTicks); result.Tokens = tokens; result.Comments = commentTokens; } catch (Exception ex) { Logger.LogError(new ParsingException(filePath, ex)); if (result == null) { result = Create(null); } } } else { result = Create(null); } result.FileName = filePath; result.FileData = sourceCodeFile.Code; return(result); }
static void Try(string ffn) { var sourceCode = System.IO.File.ReadAllText(ffn); List <IToken> codeTokens = new List <IToken>(); List <IToken> commentTokens = new List <IToken>(); Lexer preprocessorLexer = new CSharpLexer(new AntlrInputStream(sourceCode)); // Collect all tokens with lexer (CSharpLexer.g4). var tokens = preprocessorLexer.GetAllTokens(); var directiveTokens = new List <IToken>(); var directiveTokenSource = new ListTokenSource(directiveTokens); var directiveTokenStream = new CommonTokenStream(directiveTokenSource, CSharpLexer.DIRECTIVE); CSharpPreprocessorParser preprocessorParser = new CSharpPreprocessorParser(directiveTokenStream); int index = 0; bool compiliedTokens = true; while (index < tokens.Count) { var token = tokens[index]; if (token.Type == CSharpLexer.SHARP) { directiveTokens.Clear(); int directiveTokenIndex = index + 1; // Collect all preprocessor directive tokens. while (directiveTokenIndex < tokens.Count && tokens[directiveTokenIndex].Type != CSharpLexer.Eof && tokens[directiveTokenIndex].Type != CSharpLexer.DIRECTIVE_NEW_LINE && tokens[directiveTokenIndex].Type != CSharpLexer.SHARP) { if (tokens[directiveTokenIndex].Channel == CSharpLexer.COMMENTS_CHANNEL) { commentTokens.Add(tokens[directiveTokenIndex]); } else if (tokens[directiveTokenIndex].Channel != Lexer.Hidden) { directiveTokens.Add(tokens[directiveTokenIndex]); } directiveTokenIndex++; } directiveTokenSource = new ListTokenSource(directiveTokens); directiveTokenStream = new CommonTokenStream(directiveTokenSource, CSharpLexer.DIRECTIVE); preprocessorParser.TokenStream = directiveTokenStream; preprocessorParser.Reset(); // Parse condition in preprocessor directive (based on CSharpPreprocessorParser.g4 grammar). CSharpPreprocessorParser.Preprocessor_directiveContext directive = preprocessorParser.preprocessor_directive(); // if true than next code is valid and not ignored. compiliedTokens = directive.value; var directiveStr = tokens[index + 1].Text.Trim(); if ("line".Equals(directiveStr) || "error".Equals(directiveStr) || "warning".Equals(directiveStr) || "define".Equals(directiveStr) || "endregion".Equals(directiveStr) || "endif".Equals(directiveStr) || "pragma".Equals(directiveStr)) { compiliedTokens = true; } string conditionalSymbol = null; if ("define".Equals(tokens[index + 1].Text)) { // add to the conditional symbols conditionalSymbol = tokens[index + 2].Text; preprocessorParser.ConditionalSymbols.Add(conditionalSymbol); } if ("undef".Equals(tokens[index + 1].Text)) { conditionalSymbol = tokens[index + 2].Text; preprocessorParser.ConditionalSymbols.Remove(conditionalSymbol); } index = directiveTokenIndex - 1; } else if (token.Channel == CSharpLexer.COMMENTS_CHANNEL) { commentTokens.Add(token); // Colect comment tokens (if required). } else if (token.Channel != Lexer.Hidden && token.Type != CSharpLexer.DIRECTIVE_NEW_LINE && compiliedTokens) { codeTokens.Add(token); // Collect code tokens. } index++; } // At second stage tokens parsed in usual way. var codeTokenSource = new ListTokenSource(codeTokens); var codeTokenStream = new CommonTokenStream(codeTokenSource); CSharpParser parser = new CSharpParser(codeTokenStream); // Parse syntax tree (CSharpParser.g4) var listener = new ErrorListener <IToken>(parser, preprocessorLexer, codeTokenStream); parser.AddErrorListener(listener); var tree = parser.compilation_unit(); if (listener.had_error) { System.Console.WriteLine("error in parse."); } else { System.Console.WriteLine("parse completed."); } System.Console.WriteLine(codeTokenStream.OutputTokens(preprocessorLexer)); System.Console.WriteLine(tree.OutputTree(codeTokenStream, preprocessorLexer)); }
private void ProcessGrammarFile(Grammar grammar, string grammarFileName, AntlrErrorListener antlrErrorListener, CancellationToken cancellationToken) { string code = File.ReadAllText(Path.Combine(grammar.Directory, grammarFileName)); var inputStream = new AntlrInputStream(code); var codeSource = new CodeSource(grammarFileName, inputStream.ToString()); _result.GrammarFilesData.Add(grammarFileName, codeSource); string extension = Path.GetExtension(grammarFileName); if (extension != Grammar.AntlrDotExt) { return; } antlrErrorListener.CodeSource = codeSource; var antlr4Lexer = new ANTLRv4Lexer(inputStream); antlr4Lexer.RemoveErrorListeners(); antlr4Lexer.AddErrorListener(antlrErrorListener); var tokens = antlr4Lexer.GetAllTokens(); var codeTokenSource = new ListTokenSource(tokens); cancellationToken.ThrowIfCancellationRequested(); var codeTokenStream = new CommonTokenStream(codeTokenSource); var antlr4Parser = new ANTLRv4Parser(codeTokenStream); antlr4Parser.RemoveErrorListeners(); antlr4Parser.AddErrorListener(antlrErrorListener); var tree = antlr4Parser.grammarSpec(); var grammarInfoCollectorListener = new GrammarInfoCollectorListener(); grammarInfoCollectorListener.CollectInfo(antlrErrorListener.CodeSource, tree); var shortFileName = Path.GetFileNameWithoutExtension(grammarFileName); _result.GrammarActionsTextSpan[grammarFileName] = grammarInfoCollectorListener.CodeInsertions; var grammarType = grammarInfoCollectorListener.GrammarType; if (grammarType == GrammarType.Lexer || grammarType == GrammarType.Combined) { _result.LexerSuperClass = grammarInfoCollectorListener.SuperClass; } if (grammarType == GrammarType.Separated || grammarType == GrammarType.Combined) { _result.ParserSuperClass = grammarInfoCollectorListener.SuperClass; _result.Rules = grammarInfoCollectorListener.Rules; } void ErrorAction(ParsingError parsingError) { ErrorEvent?.Invoke(this, parsingError); _result.Errors.Add(parsingError); } var caseInsensitiveTypeOptionMatcher = new CaseInsensitiveTypeOptionMatcher(codeSource, grammarType, ErrorAction); var runtimeOptionMatcher = new RuntimeOptionMatcher(codeSource, grammarType, ErrorAction); var visitorOptionMatcher = new VisitorOptionMatcher(codeSource, grammarType, ErrorAction); var listenerOptionMatcher = new ListenerOptionMatcher(codeSource, grammarType, ErrorAction); var packageOptionMatcher = new PackageOptionMatcher(codeSource, grammarType, ErrorAction); var rootOptionMatcher = new RootOptionMatcher(codeSource, grammarType, ErrorAction, _result.Rules); var predictionOptionMatcher = new PredictionModeOptionMatcher(codeSource, grammarType, ErrorAction); foreach (IToken token in tokens) { if (token.Type == ANTLRv4Lexer.LINE_COMMENT || token.Type == ANTLRv4Lexer.BLOCK_COMMENT) { if (caseInsensitiveTypeOptionMatcher.Match(token, out var caseInsensitiveType)) { _result.CaseInsensitiveType = caseInsensitiveType; continue; } if (runtimeOptionMatcher.Match(token, out Runtime runtime)) { _result.Runtime = runtime; continue; } if (packageOptionMatcher.Match(token, out string package)) { _result.Package = package; continue; } if (visitorOptionMatcher.Match(token, out bool generateVisitor)) { _result.Visitor = generateVisitor; continue; } if (listenerOptionMatcher.Match(token, out bool generateListener)) { _result.Listener = generateListener; continue; } if (rootOptionMatcher.Match(token, out string root)) { _result.Root = root; continue; } if (predictionOptionMatcher.Match(token, out PredictionMode predictionMode)) { _result.PredictionMode = predictionMode; continue; } } } }
public static void Main(string[] args) { try { string[] files = Directory.GetFiles(".", "*.cs"); foreach (string file in files) { Console.WriteLine(file + "---------------------------------------"); //Read the file string text = File.ReadAllText(file); //Create the lexer CSharpLexer lexer = new CSharpLexer(new AntlrInputStream(text)); var tokens = lexer.GetAllTokens(); List <IToken> codeTokens = new List <IToken>(); List <IToken> commentTokens = new List <IToken>(); var directiveTokens = new List <IToken>(); var directiveTokenSource = new ListTokenSource(directiveTokens); var directiveTokenStream = new CommonTokenStream(directiveTokenSource, CSharpLexer.DIRECTIVE); CSharpPreprocessorParser preprocessorParser = new CSharpPreprocessorParser(directiveTokenStream); int index = 0; bool compiliedTokens = true; while (index < tokens.Count) { var token = tokens[index]; if (token.Type == CSharpLexer.SHARP) { directiveTokens.Clear(); int directiveTokenIndex = index + 1; // Collect all preprocessor directive tokens. while (directiveTokenIndex < tokens.Count && tokens[directiveTokenIndex].Type != CSharpLexer.Eof && tokens[directiveTokenIndex].Type != CSharpLexer.DIRECTIVE_NEW_LINE && tokens[directiveTokenIndex].Type != CSharpLexer.SHARP) { if (tokens[directiveTokenIndex].Channel == CSharpLexer.COMMENTS_CHANNEL) { commentTokens.Add(tokens[directiveTokenIndex]); } else if (tokens[directiveTokenIndex].Channel != Lexer.Hidden) { //Console.WriteLine(allTokens[directiveTokenIndex] + " HOLA"); directiveTokens.Add(tokens[directiveTokenIndex]); } directiveTokenIndex++; } directiveTokenSource = new ListTokenSource(directiveTokens); directiveTokenStream = new CommonTokenStream(directiveTokenSource, CSharpLexer.DIRECTIVE); preprocessorParser.TokenStream = directiveTokenStream; //preprocessorParser.SetInputStream(directiveTokenStream); preprocessorParser.Reset(); // Parse condition in preprocessor directive (based on CSharpPreprocessorParser.g4 grammar). CSharpPreprocessorParser.Preprocessor_directiveContext directive = preprocessorParser.preprocessor_directive(); // if true than next code is valid and not ignored. compiliedTokens = directive.value; String directiveStr = tokens[index + 1].Text.Trim(); if ("line".Equals(directiveStr) || "error".Equals(directiveStr) || "warning".Equals(directiveStr) || "define".Equals(directiveStr) || "endregion".Equals(directiveStr) || "endif".Equals(directiveStr) || "pragma".Equals(directiveStr)) { //Console.WriteLine(directiveStr); compiliedTokens = true; } String conditionalSymbol = null; if ("define".Equals(tokens[index + 1].Text)) { // add to the conditional symbols conditionalSymbol = tokens[index + 2].Text; preprocessorParser.ConditionalSymbols.Add(conditionalSymbol); } if ("undef".Equals(tokens[index + 1].Text)) { conditionalSymbol = tokens[index + 2].Text; preprocessorParser.ConditionalSymbols.Remove(conditionalSymbol); } //This code deletes the directive tokens from the input so that they don't interfere with the parsing process // In all of the cases, we have to remove at least two positions of the tokens array tokens.RemoveAt(directiveTokenIndex - 1); tokens.RemoveAt(directiveTokenIndex - 2); if ("pragma".Equals(directiveStr) || "warning".Equals(directiveStr) || "region".Equals(directiveStr) || "error".Equals(directiveStr)) { // Remove three positions before tokens.RemoveAt(directiveTokenIndex - 3); directiveTokenIndex--; } else if ("define".Equals(directiveStr) || "undef".Equals(directiveStr) || "if".Equals(directiveStr) || "elif".Equals(directiveStr) || "line".Equals(directiveStr)) { // Remove four positions before tokens.RemoveAt(directiveTokenIndex - 3); tokens.RemoveAt(directiveTokenIndex - 4); directiveTokenIndex -= 2; } directiveTokenIndex -= 2; index = directiveTokenIndex - 1; } else if (token.Channel == CSharpLexer.COMMENTS_CHANNEL) { commentTokens.Add(token); // Colect comment tokens (if required). } else if (token.Channel != Lexer.Hidden && token.Type != CSharpLexer.DIRECTIVE_NEW_LINE && compiliedTokens) { codeTokens.Add(token); // Collect code tokens. } index++; } // At second stage tokens parsed in usual way. var codeTokenSource = new ListTokenSource(tokens); var codeTokenStream = new CommonTokenStream(codeTokenSource); CSharpParser parser = new CSharpParser(codeTokenStream); ////Create the token stream //CommonTokenStream tokens = new CommonTokenStream(lexer); //CSharpParser parser = new CSharpParser(tokens); IParseTree tree = parser.compilation_unit(); ////Walk the tree ParseTreeWalker walker = new ParseTreeWalker(); walker.Walk(new ProgressPrinter(), tree); } } catch (Exception e) { Console.WriteLine("Error (Program.cs): " + e); } }
public GrammarCheckedState Check(InputState inputState, CancellationToken cancellationToken = default) { var grammar = inputState.Grammar; var result = new GrammarCheckedState(inputState); try { var antlrErrorListener = new AntlrErrorListener(); antlrErrorListener.ErrorEvent += ErrorEvent; antlrErrorListener.ErrorEvent += (sender, error) => { lock (result.Errors) { result.Errors.Add(error); } }; foreach (string grammarFileName in grammar.Files) { string code = File.ReadAllText(Path.Combine(grammar.Directory, grammarFileName)); var inputStream = new AntlrInputStream(code); var codeSource = new CodeSource(grammarFileName, inputStream.ToString()); result.GrammarFilesData.Add(grammarFileName, codeSource); string extension = Path.GetExtension(grammarFileName); if (extension != Grammar.AntlrDotExt) { continue; } antlrErrorListener.CodeSource = codeSource; var antlr4Lexer = new ANTLRv4Lexer(inputStream); antlr4Lexer.RemoveErrorListeners(); antlr4Lexer.AddErrorListener(antlrErrorListener); var codeTokenSource = new ListTokenSource(antlr4Lexer.GetAllTokens()); cancellationToken.ThrowIfCancellationRequested(); var codeTokenStream = new CommonTokenStream(codeTokenSource); var antlr4Parser = new ANTLRv4Parser(codeTokenStream); antlr4Parser.RemoveErrorListeners(); antlr4Parser.AddErrorListener(antlrErrorListener); var tree = antlr4Parser.grammarSpec(); var grammarInfoCollectorListener = new GrammarInfoCollectorListener(); grammarInfoCollectorListener.CollectInfo(antlrErrorListener.CodeSource, tree); var shortFileName = Path.GetFileNameWithoutExtension(grammarFileName); result.GrammarActionsTextSpan[grammarFileName] = grammarInfoCollectorListener.CodeInsertions; if (grammarFileName.Contains(Grammar.LexerPostfix)) { result.LexerSuperClass = grammarInfoCollectorListener.SuperClass; } if (grammarFileName.Contains(Grammar.ParserPostfix)) { result.ParserSuperClass = grammarInfoCollectorListener.SuperClass; } if (!shortFileName.Contains(Grammar.LexerPostfix)) { result.Rules = grammarInfoCollectorListener.Rules; cancellationToken.ThrowIfCancellationRequested(); } } } catch (Exception ex) { result.Exception = ex; if (!(ex is OperationCanceledException)) { ErrorEvent?.Invoke(this, new ParsingError(ex, WorkflowStage.GrammarChecked)); } } return(result); }