public LeftRecursiveRule(Grammar g, string name, RuleAST ast) : base(g, name, ast, 1) { originalAST = ast; alt = new Alternative[numberOfAlts + 1]; // always just one for (int i = 1; i <= numberOfAlts; i++) alt[i] = new Alternative(this, i); }
public override void DiscoverRule(RuleAST rule, GrammarAST ID, IList<GrammarAST> modifiers, ActionAST arg, ActionAST returns, GrammarAST thrws, GrammarAST options, ActionAST locals, IList<GrammarAST> actions, GrammarAST block) { currentRule = g.GetRule(ID.Text); }
public override void DiscoverRule(RuleAST rule, GrammarAST ID, IList<GrammarAST> modifiers, ActionAST arg, ActionAST returns, GrammarAST thrws, GrammarAST options, ActionAST locals, IList<GrammarAST> actions, GrammarAST block) { int numAlts = block.ChildCount; Rule r; if (LeftRecursiveRuleAnalyzer.HasImmediateRecursiveRuleRefs(rule, ID.Text)) { r = new LeftRecursiveRule(g, ID.Text, rule); } else { r = new Rule(g, ID.Text, rule, numAlts); } rules[r.name] = r; if (arg != null) { r.args = ScopeParser.ParseTypedArgList(arg, arg.Text, g); r.args.type = AttributeDict.DictType.ARG; r.args.ast = arg; arg.resolver = r.alt[currentOuterAltNumber]; } if (returns != null) { r.retvals = ScopeParser.ParseTypedArgList(returns, returns.Text, g); r.retvals.type = AttributeDict.DictType.RET; r.retvals.ast = returns; } if (locals != null) { r.locals = ScopeParser.ParseTypedArgList(locals, locals.Text, g); r.locals.type = AttributeDict.DictType.LOCAL; r.locals.ast = locals; } foreach (GrammarAST a in actions) { // a = ^(AT ID ACTION) ActionAST action = (ActionAST)a.GetChild(1); r.namedActions[a.GetChild(0).Text] = action; action.resolver = r; } }
/** Make sure even imaginary nodes know the input stream */ public override object Create(int tokenType, string text) { GrammarAST t; if (tokenType == ANTLRParser.RULE) { // needed by TreeWizard to make RULE tree t = new RuleAST(new CommonToken(tokenType, text)); } else if (tokenType == ANTLRParser.STRING_LITERAL) { // implicit lexer construction done with wizard; needs this node type // whereas grammar ANTLRParser.g can use token option to spec node type t = new TerminalAST(new CommonToken(tokenType, text)); } else { t = (GrammarAST)base.Create(tokenType, text); } t.Token.InputStream = input; return t; }
public override void DiscoverLexerRule(RuleAST rule, GrammarAST ID, IList<GrammarAST> modifiers, GrammarAST block) { currentRule = g.GetRule(ID.Text); }
public RuleAST(RuleAST node) : base(node) { }
public int actionIndex = -1; // if lexer; 0..n-1 for n actions in a rule public Rule(Grammar g, string name, RuleAST ast, int numberOfAlts) { this.g = g; this.name = name; this.ast = ast; this.numberOfAlts = numberOfAlts; alt = new Alternative[numberOfAlts + 1]; // 1..n for (int i = 1; i <= numberOfAlts; i++) alt[i] = new Alternative(this, i); }
public override void FinishRule(RuleAST rule, GrammarAST ID, GrammarAST block) { if (rule.IsLexerRule()) return; BlockAST blk = (BlockAST)rule.GetFirstChildWithType(BLOCK); int nalts = blk.ChildCount; GrammarAST idAST = (GrammarAST)rule.GetChild(0); for (int i = 0; i < nalts; i++) { AltAST altAST = (AltAST)blk.GetChild(i); if (altAST.altLabel != null) { string altLabel = altAST.altLabel.Text; // first check that label doesn't conflict with a rule // label X or x can't be rule x. Rule r; if (ruleCollector.rules.TryGetValue(Utils.Decapitalize(altLabel), out r) && r != null) { g.tool.errMgr.GrammarError(ErrorType.ALT_LABEL_CONFLICTS_WITH_RULE, g.fileName, altAST.altLabel.Token, altLabel, r.name); } // Now verify that label X or x doesn't conflict with label // in another rule. altLabelToRuleName has both X and x mapped. string prevRuleForLabel; if (ruleCollector.altLabelToRuleName.TryGetValue(altLabel, out prevRuleForLabel) && prevRuleForLabel != null && !prevRuleForLabel.Equals(rule.GetRuleName())) { g.tool.errMgr.GrammarError(ErrorType.ALT_LABEL_REDEF, g.fileName, altAST.altLabel.Token, altLabel, rule.GetRuleName(), prevRuleForLabel); } } } IList<GrammarAST> altLabels; int numAltLabels = 0; if (ruleCollector.ruleToAltLabels.TryGetValue(rule.GetRuleName(), out altLabels) && altLabels != null) numAltLabels = altLabels.Count; if (numAltLabels > 0 && nalts != numAltLabels) { g.tool.errMgr.GrammarError(ErrorType.RULE_WITH_TOO_FEW_ALT_LABELS, g.fileName, idAST.Token, rule.GetRuleName()); } }
public override void DiscoverLexerRule(RuleAST rule, GrammarAST ID, IList<GrammarAST> modifiers, GrammarAST block) { CheckInvalidRuleDef(ID.Token); if (modifiers != null) { foreach (GrammarAST tree in modifiers) { if (tree.Type == ANTLRParser.FRAGMENT) { inFragmentRule = true; } } } if (!inFragmentRule) { nonFragmentRuleCount++; } }
public override void DiscoverRule(RuleAST rule, GrammarAST ID, IList<GrammarAST> modifiers, ActionAST arg, ActionAST returns, GrammarAST thrws, GrammarAST options, ActionAST locals, IList<GrammarAST> actions, GrammarAST block) { // TODO: chk that all or no alts have "# label" CheckInvalidRuleDef(ID.Token); }
public virtual IDictionary<string, IList<System.Tuple<int, AltAST>>> GetLabeledAlternatives(RuleAST ast) { AltLabelVisitor visitor = new AltLabelVisitor(new Antlr.Runtime.Tree.CommonTreeNodeStream(new GrammarASTAdaptor(), ast)); visitor.rule(); return visitor.GetLabeledAlternatives(); }
public virtual IList<AltAST> GetUnlabeledAlternatives(RuleAST ast) { AltLabelVisitor visitor = new AltLabelVisitor(new Antlr.Runtime.Tree.CommonTreeNodeStream(new GrammarASTAdaptor(), ast)); visitor.rule(); return visitor.GetUnlabeledAlternatives(); }
/** * <pre> * (RULE e int _p (returns int v) * (BLOCK * (ALT * (BLOCK * (ALT INT {$v = $INT.int;}) * (ALT '(' (= x e) ')' {$v = $x.v;}) * (ALT ID)) * (* (BLOCK * (OPTIONS ...) * (ALT {7 >= $_p}? '*' (= b e) {$v = $a.v * $b.v;}) * (ALT {6 >= $_p}? '+' (= b e) {$v = $a.v + $b.v;}) * (ALT {3 >= $_p}? '++') (ALT {2 >= $_p}? '--')))))) * </pre> */ public virtual void SetAltASTPointers(LeftRecursiveRule r, RuleAST t) { //System.Console.WriteLine("RULE: " + t.ToStringTree()); BlockAST ruleBlk = (BlockAST)t.GetFirstChildWithType(ANTLRParser.BLOCK); AltAST mainAlt = (AltAST)ruleBlk.GetChild(0); BlockAST primaryBlk = (BlockAST)mainAlt.GetChild(0); BlockAST opsBlk = (BlockAST)mainAlt.GetChild(1).GetChild(0); // (* BLOCK ...) for (int i = 0; i < r.recPrimaryAlts.Count; i++) { LeftRecursiveRuleAltInfo altInfo = r.recPrimaryAlts[i]; altInfo.altAST = (AltAST)primaryBlk.GetChild(i); altInfo.altAST.leftRecursiveAltInfo = altInfo; altInfo.originalAltAST.leftRecursiveAltInfo = altInfo; //altInfo.originalAltAST.Parent = altInfo.altAST.Parent; //System.Console.WriteLine(altInfo.altAST.ToStringTree()); } for (int i = 0; i < r.recOpAlts.Count; i++) { LeftRecursiveRuleAltInfo altInfo = r.recOpAlts.GetElement(i); altInfo.altAST = (AltAST)opsBlk.GetChild(i); altInfo.altAST.leftRecursiveAltInfo = altInfo; altInfo.originalAltAST.leftRecursiveAltInfo = altInfo; //altInfo.originalAltAST.Parent = altInfo.altAST.Parent; //System.Console.WriteLine(altInfo.altAST.ToStringTree()); } }
/** Build lexer grammar from combined grammar that looks like: * * (COMBINED_GRAMMAR A * (tokens { X (= Y 'y')) * (OPTIONS (= x 'y')) * (@ members {foo}) * (@ lexer header {package jj;}) * (RULES (RULE .+))) * * Move rules and actions to new tree, don't dup. Split AST apart. * We'll have this Grammar share token symbols later; don't generate * tokenVocab or tokens{} section. Copy over named actions. * * Side-effects: it removes children from GRAMMAR & RULES nodes * in combined AST. Anything cut out is dup'd before * adding to lexer to avoid "who's ur daddy" issues */ public virtual GrammarRootAST ExtractImplicitLexer(Grammar combinedGrammar) { GrammarRootAST combinedAST = combinedGrammar.ast; //tool.log("grammar", "before="+combinedAST.toStringTree()); GrammarASTAdaptor adaptor = new GrammarASTAdaptor(combinedAST.Token.InputStream); GrammarAST[] elements = combinedAST.GetChildrenAsArray(); // MAKE A GRAMMAR ROOT and ID string lexerName = combinedAST.GetChild(0).Text + "Lexer"; GrammarRootAST lexerAST = new GrammarRootAST(new CommonToken(ANTLRParser.GRAMMAR, "LEXER_GRAMMAR"), combinedGrammar.ast.tokenStream); lexerAST.grammarType = ANTLRParser.LEXER; lexerAST.Token.InputStream = combinedAST.Token.InputStream; lexerAST.AddChild((ITree)adaptor.Create(ANTLRParser.ID, lexerName)); // COPY OPTIONS GrammarAST optionsRoot = (GrammarAST)combinedAST.GetFirstChildWithType(ANTLRParser.OPTIONS); if (optionsRoot != null && optionsRoot.ChildCount != 0) { GrammarAST lexerOptionsRoot = (GrammarAST)adaptor.DupNode(optionsRoot); lexerAST.AddChild(lexerOptionsRoot); GrammarAST[] options = optionsRoot.GetChildrenAsArray(); foreach (GrammarAST o in options) { string optionName = o.GetChild(0).Text; if (Grammar.lexerOptions.Contains(optionName) && !Grammar.doNotCopyOptionsToLexer.Contains(optionName)) { GrammarAST optionTree = (GrammarAST)adaptor.DupTree(o); lexerOptionsRoot.AddChild(optionTree); lexerAST.SetOption(optionName, (GrammarAST)optionTree.GetChild(1)); } } } // COPY all named actions, but only move those with lexer:: scope IList<GrammarAST> actionsWeMoved = new List<GrammarAST>(); foreach (GrammarAST e in elements) { if (e.Type == ANTLRParser.AT) { lexerAST.AddChild((ITree)adaptor.DupTree(e)); if (e.GetChild(0).Text.Equals("lexer")) { actionsWeMoved.Add(e); } } } foreach (GrammarAST r in actionsWeMoved) { combinedAST.DeleteChild(r); } GrammarAST combinedRulesRoot = (GrammarAST)combinedAST.GetFirstChildWithType(ANTLRParser.RULES); if (combinedRulesRoot == null) return lexerAST; // MOVE lexer rules GrammarAST lexerRulesRoot = (GrammarAST)adaptor.Create(ANTLRParser.RULES, "RULES"); lexerAST.AddChild(lexerRulesRoot); IList<GrammarAST> rulesWeMoved = new List<GrammarAST>(); GrammarASTWithOptions[] rules; if (combinedRulesRoot.ChildCount > 0) { rules = combinedRulesRoot.Children.Cast<GrammarASTWithOptions>().ToArray(); } else { rules = new GrammarASTWithOptions[0]; } foreach (GrammarASTWithOptions r in rules) { string ruleName = r.GetChild(0).Text; if (Grammar.IsTokenName(ruleName)) { lexerRulesRoot.AddChild((ITree)adaptor.DupTree(r)); rulesWeMoved.Add(r); } } foreach (GrammarAST r in rulesWeMoved) { combinedRulesRoot.DeleteChild(r); } // Will track 'if' from IF : 'if' ; rules to avoid defining new token for 'if' IList<System.Tuple<GrammarAST, GrammarAST>> litAliases = Grammar.GetStringLiteralAliasesFromLexerRules(lexerAST); ISet<string> stringLiterals = combinedGrammar.GetStringLiterals(); // add strings from combined grammar (and imported grammars) into lexer // put them first as they are keywords; must resolve ambigs to these rules // tool.log("grammar", "strings from parser: "+stringLiterals); int insertIndex = 0; foreach (string lit in stringLiterals) { // if lexer already has a rule for literal, continue if (litAliases != null) { foreach (System.Tuple<GrammarAST, GrammarAST> pair in litAliases) { GrammarAST litAST = pair.Item2; if (lit.Equals(litAST.Text)) goto continueNextLit; } } // create for each literal: (RULE <uniquename> (BLOCK (ALT <lit>)) string rname = combinedGrammar.GetStringLiteralLexerRuleName(lit); // can't use wizard; need special node types GrammarAST litRule = new RuleAST(ANTLRParser.RULE); BlockAST blk = new BlockAST(ANTLRParser.BLOCK); AltAST alt = new AltAST(ANTLRParser.ALT); TerminalAST slit = new TerminalAST(new CommonToken(ANTLRParser.STRING_LITERAL, lit)); alt.AddChild(slit); blk.AddChild(alt); CommonToken idToken = new CommonToken(ANTLRParser.TOKEN_REF, rname); litRule.AddChild(new TerminalAST(idToken)); litRule.AddChild(blk); lexerRulesRoot.InsertChild(insertIndex, litRule); // lexerRulesRoot.getChildren().add(0, litRule); lexerRulesRoot.FreshenParentAndChildIndexes(); // reset indexes and set litRule parent // next literal will be added after the one just added insertIndex++; continueNextLit: ; } // TODO: take out after stable if slow lexerAST.SanityCheckParentAndChildIndexes(); combinedAST.SanityCheckParentAndChildIndexes(); // tool.log("grammar", combinedAST.toTokenString()); combinedGrammar.tool.Log("grammar", "after extract implicit lexer =" + combinedAST.ToStringTree()); combinedGrammar.tool.Log("grammar", "lexer =" + lexerAST.ToStringTree()); if (lexerRulesRoot.ChildCount == 0) return null; return lexerAST; }
public override void DiscoverLexerRule(RuleAST rule, GrammarAST ID, IList<GrammarAST> modifiers, GrammarAST block) { int numAlts = block.ChildCount; Rule r = new Rule(g, ID.Text, rule, numAlts); r.mode = currentModeName; if (modifiers.Count > 0) r.modifiers = modifiers; rules[r.name] = r; }