public virtual RuleAST ParseArtificialRule(Grammar g, string ruleText) { ANTLRLexer lexer = new ANTLRLexer(new ANTLRStringStream(ruleText)); GrammarASTAdaptor adaptor = new GrammarASTAdaptor(lexer.CharStream); CommonTokenStream tokens = new CommonTokenStream(lexer); lexer.tokens = tokens; ToolANTLRParser p = new ToolANTLRParser(tokens, tool); p.TreeAdaptor = adaptor; IToken ruleStart = null; try { Antlr.Runtime.AstParserRuleReturnScope <GrammarAST, IToken> r = p.rule(); RuleAST tree = (RuleAST)r.Tree; ruleStart = r.Start; GrammarTransformPipeline.SetGrammarPtr(g, tree); GrammarTransformPipeline.AugmentTokensWithOriginalPosition(g, tree); return(tree); } catch (Exception e) { tool.errMgr.ToolError(ErrorType.INTERNAL_ERROR, e, ruleStart, "error parsing rule created during left-recursion detection: " + ruleText); } return(null); }
private List <LabelInfo> FindLabelsInScope(SnapshotPoint triggerPoint) { List <LabelInfo> labels = new List <LabelInfo>(); /* use the experimental model to locate and process the expression */ Stopwatch stopwatch = Stopwatch.StartNew(); // lex the entire document var currentSnapshot = triggerPoint.Snapshot; var input = new SnapshotCharStream(currentSnapshot, new Span(0, currentSnapshot.Length)); var lexer = new ANTLRLexer(input); var tokens = new CommonTokenStream(lexer); tokens.Fill(); // locate the last token before the trigger point while (true) { IToken nextToken = tokens.Lt(1); if (nextToken.Type == CharStreamConstants.EndOfFile) { break; } if (nextToken.StartIndex > triggerPoint.Position) { break; } tokens.Consume(); } bool inAction = false; IToken triggerToken = tokens.LT(-1); switch (triggerToken.Type) { case ANTLRLexer.RULE_REF: case ANTLRLexer.TOKEN_REF: case ANTLRLexer.DOLLAR: break; case ANTLRLexer.ACTION: case ANTLRLexer.FORCED_ACTION: case ANTLRLexer.SEMPRED: case ANTLRLexer.ARG_ACTION: inAction = true; break; default: return(labels); } NetworkInterpreter interpreter = CreateNetworkInterpreter(tokens); while (interpreter.TryStepBackward()) { if (interpreter.Contexts.Count == 0 || interpreter.Contexts.Count > 4000) { break; } if (interpreter.Contexts.All(context => context.BoundedStart)) { break; } } if (interpreter.Failed) { interpreter.Contexts.Clear(); } interpreter.CombineBoundedStartContexts(); HashSet <IToken> labelTokens = new HashSet <IToken>(TokenIndexEqualityComparer.Default); foreach (var context in interpreter.Contexts) { var tokenTransitions = context.Transitions.Where(i => i.TokenIndex != null).ToList(); for (int i = 1; i < tokenTransitions.Count - 1; i++) { if (tokenTransitions[i].Symbol != ANTLRLexer.TOKEN_REF && tokenTransitions[i].Symbol != ANTLRLexer.RULE_REF) { continue; } // we add explicit labels, plus implicit labels if we're in an action if (tokenTransitions[i + 1].Symbol == ANTLRLexer.ASSIGN || tokenTransitions[i + 1].Symbol == ANTLRLexer.PLUS_ASSIGN) { RuleBinding rule = interpreter.Network.StateRules[tokenTransitions[i + 1].Transition.SourceState.Id]; if (rule.Name == AntlrAtnBuilder.RuleNames.TreeRoot || rule.Name == AntlrAtnBuilder.RuleNames.ElementNoOptionSpec) { labelTokens.Add(tokenTransitions[i].Token); } } else if (inAction && tokenTransitions[i - 1].Symbol != ANTLRLexer.ASSIGN && tokenTransitions[i - 1].Symbol != ANTLRLexer.PLUS_ASSIGN) { RuleBinding rule = interpreter.Network.StateRules[tokenTransitions[i].Transition.SourceState.Id]; if (rule.Name == AntlrAtnBuilder.RuleNames.Terminal || rule.Name == AntlrAtnBuilder.RuleNames.NotTerminal || rule.Name == AntlrAtnBuilder.RuleNames.RuleRef) { labelTokens.Add(tokenTransitions[i].Token); } } } } foreach (var token in labelTokens) { labels.Add(new LabelInfo(token.Text, "(label) " + token.Text, new SnapshotSpan(triggerPoint.Snapshot, Span.FromBounds(token.StartIndex, token.StopIndex + 1)), StandardGlyphGroup.GlyphGroupField, Enumerable.Empty <LabelInfo>())); } /* add scopes */ if (inAction) { /* add global scopes */ IList <IToken> tokensList = tokens.GetTokens(); for (int i = 0; i < tokensList.Count - 1; i++) { var token = tokensList[i]; /* all global scopes appear before the first rule. before the first rule, the only place a ':' can appear is * in the form '::' for things like @lexer::namespace{} */ if (token.Type == ANTLRLexer.COLON && tokensList[i + 1].Type == ANTLRLexer.COLON) { break; } if (token.Type == ANTLRLexer.SCOPE) { var nextToken = tokensList.Skip(i + 1).FirstOrDefault(t => t.Channel == TokenChannels.Default); if (nextToken != null && (nextToken.Type == ANTLRLexer.RULE_REF || nextToken.Type == ANTLRLexer.TOKEN_REF)) { // TODO: parse scope members IToken actionToken = tokensList.Skip(nextToken.TokenIndex + 1).FirstOrDefault(t => t.Channel == TokenChannels.Default); IEnumerable <LabelInfo> members = Enumerable.Empty <LabelInfo>(); if (actionToken != null && actionToken.Type == ANTLRLexer.ACTION) { IEnumerable <IToken> scopeMembers = ExtractScopeAttributes(nextToken); members = scopeMembers.Select(member => { string name = member.Text; SnapshotSpan definition = new SnapshotSpan(triggerPoint.Snapshot, Span.FromBounds(member.StartIndex, member.StopIndex + 1)); StandardGlyphGroup glyph = StandardGlyphGroup.GlyphGroupField; IEnumerable <LabelInfo> nestedMembers = Enumerable.Empty <LabelInfo>(); return(new LabelInfo(name, string.Empty, definition, glyph, nestedMembers)); }); } labels.Add(new LabelInfo(nextToken.Text, "(global scope) " + nextToken.Text, new SnapshotSpan(triggerPoint.Snapshot, Span.FromBounds(nextToken.StartIndex, nextToken.StopIndex + 1)), StandardGlyphGroup.GlyphGroupNamespace, members)); } } } /* add rule scopes */ // todo } /* add arguments and return values */ if (inAction) { HashSet <IToken> argumentTokens = new HashSet <IToken>(TokenIndexEqualityComparer.Default); foreach (var context in interpreter.Contexts) { var tokenTransitions = context.Transitions.Where(i => i.TokenIndex != null).ToList(); for (int i = 1; i < tokenTransitions.Count; i++) { if (tokenTransitions[i].Symbol == ANTLRLexer.RETURNS || tokenTransitions[i].Symbol == ANTLRLexer.COLON) { break; } if (tokenTransitions[i].Symbol == ANTLRLexer.ARG_ACTION) { argumentTokens.Add(tokenTransitions[i].Token); } } } foreach (var token in argumentTokens) { IEnumerable <IToken> arguments = ExtractArguments(token); foreach (var argument in arguments) { labels.Add(new LabelInfo(argument.Text, "(parameter) " + argument.Text, new SnapshotSpan(triggerPoint.Snapshot, Span.FromBounds(argument.StartIndex, argument.StopIndex + 1)), StandardGlyphGroup.GlyphGroupVariable, Enumerable.Empty <LabelInfo>())); } } } /* add return values */ if (inAction) { HashSet <IToken> returnTokens = new HashSet <IToken>(TokenIndexEqualityComparer.Default); foreach (var context in interpreter.Contexts) { var tokenTransitions = context.Transitions.Where(i => i.TokenIndex != null).ToList(); for (int i = 1; i < tokenTransitions.Count - 1; i++) { if (tokenTransitions[i].Symbol == ANTLRLexer.COLON) { break; } if (tokenTransitions[i].Symbol == ANTLRLexer.RETURNS) { if (tokenTransitions[i + 1].Symbol == ANTLRLexer.ARG_ACTION) { returnTokens.Add(tokenTransitions[i + 1].Token); } break; } } } foreach (var token in returnTokens) { IEnumerable <IToken> returnValues = ExtractArguments(token); foreach (var returnValue in returnValues) { labels.Add(new LabelInfo(returnValue.Text, "(return value) " + returnValue.Text, new SnapshotSpan(triggerPoint.Snapshot, Span.FromBounds(returnValue.StartIndex, returnValue.StopIndex + 1)), StandardGlyphGroup.GlyphGroupVariable, Enumerable.Empty <LabelInfo>())); } } } /* add intrinsic labels ($start, $type, $text, $enclosingRuleName) */ IToken ruleNameToken = null; HashSet <IToken> enclosingRuleNameTokens = new HashSet <IToken>(TokenIndexEqualityComparer.Default); foreach (var context in interpreter.Contexts) { var tokenTransitions = context.Transitions.Where(i => i.Symbol == ANTLRLexer.RULE_REF || i.Symbol == ANTLRLexer.TOKEN_REF).ToList(); if (!tokenTransitions.Any()) { continue; } ruleNameToken = tokenTransitions.First().Token; if (ruleNameToken != null) { enclosingRuleNameTokens.Add(ruleNameToken); } } foreach (var token in enclosingRuleNameTokens) { // TODO: add members labels.Add(new LabelInfo(token.Text, "(enclosing rule) " + token.Text, new SnapshotSpan(triggerPoint.Snapshot, Span.FromBounds(token.StartIndex, token.StopIndex + 1)), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); } GrammarType grammarType = GrammarType.None; int mark = tokens.Mark(); try { tokens.Seek(0); bool hasGrammarType = false; while (!hasGrammarType) { int la1 = tokens.LA(1); switch (la1) { case ANTLRLexer.GRAMMAR: IToken previous = tokens.LT(-1); if (previous == null) { grammarType = GrammarType.Combined; } else if (previous.Type == ANTLRLexer.LEXER) { grammarType = GrammarType.Lexer; } else if (previous.Type == ANTLRLexer.PARSER) { grammarType = GrammarType.Parser; } else if (previous.Type == ANTLRLexer.TREE) { grammarType = GrammarType.TreeParser; } else { grammarType = GrammarType.None; } hasGrammarType = true; break; case CharStreamConstants.EndOfFile: hasGrammarType = true; break; default: break; } tokens.Consume(); } } finally { tokens.Rewind(mark); } if (inAction) { switch (grammarType) { case GrammarType.Combined: if (ruleNameToken == null) { goto default; } if (ruleNameToken.Type == ANTLRLexer.RULE_REF) { goto case GrammarType.Parser; } else { goto case GrammarType.Lexer; } case GrammarType.Lexer: labels.Add(new LabelInfo("text", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("type", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("line", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("index", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("pos", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("channel", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("start", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("stop", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("int", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); break; case GrammarType.Parser: labels.Add(new LabelInfo("text", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("start", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("stop", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("tree", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("st", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); break; case GrammarType.TreeParser: labels.Add(new LabelInfo("text", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("start", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("tree", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("st", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); break; default: // if we're unsure about the grammar type, include all the possible options to make sure we're covered labels.Add(new LabelInfo("text", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("type", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("line", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("index", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("pos", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("channel", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("start", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("stop", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("int", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("tree", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); labels.Add(new LabelInfo("st", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>())); break; } } return(labels); }
public virtual RuleAST ParseArtificialRule(Grammar g, string ruleText) { ANTLRLexer lexer = new ANTLRLexer(new ANTLRStringStream(ruleText)); GrammarASTAdaptor adaptor = new GrammarASTAdaptor(lexer.CharStream); CommonTokenStream tokens = new CommonTokenStream(lexer); lexer.tokens = tokens; ToolANTLRParser p = new ToolANTLRParser(tokens, tool); p.TreeAdaptor = adaptor; IToken ruleStart = null; try { Antlr.Runtime.AstParserRuleReturnScope<GrammarAST, IToken> r = p.rule(); RuleAST tree = (RuleAST)r.Tree; ruleStart = r.Start; GrammarTransformPipeline.SetGrammarPtr(g, tree); GrammarTransformPipeline.AugmentTokensWithOriginalPosition(g, tree); return tree; } catch (Exception e) { tool.errMgr.ToolError(ErrorType.INTERNAL_ERROR, e, ruleStart, "error parsing rule created during left-recursion detection: " + ruleText); } return null; }