public override void RuleRef(GrammarAST @ref, ActionAST arg) { if (@ref is GrammarASTWithOptions) { GrammarASTWithOptions grammarASTWithOptions = (GrammarASTWithOptions)@ref; if (bool.Parse(grammarASTWithOptions.GetOptionString(LeftFactoringRuleTransformer.SUPPRESS_ACCESSOR) ?? "false")) { return; } } frequencies.Peek().Add(RuleFunction.GetLabelName(grammar, @ref)); minFrequencies.Peek().Add(RuleFunction.GetLabelName(grammar, @ref)); }
public override void ElementOption(GrammarASTWithOptions elem, GrammarAST ID, GrammarAST valueAST) { //string v = null; bool ok = CheckElementOptions(elem, ID, valueAST); //if (ok) //{ // if (v != null) // { // t.setOption(ID.Text, v); // } // else // { // t.setOption(TerminalAST.defaultTokenOption, v); // } //} }
/** Check option is appropriate for elem; parent of ID is ELEMENT_OPTIONS */ internal virtual bool CheckElementOptions(GrammarASTWithOptions elem, GrammarAST ID, GrammarAST valueAST) { if (checkAssocElementOption && ID != null && "assoc".Equals(ID.Text)) { if (elem.Type != ANTLRParser.ALT) { IToken optionID = ID.Token; string fileName = optionID.InputStream.SourceName; g.tool.errMgr.GrammarError(ErrorType.UNRECOGNIZED_ASSOC_OPTION, fileName, optionID, currentRuleName); } } if (elem is RuleRefAST) { return(CheckRuleRefOptions((RuleRefAST)elem, ID, valueAST)); } if (elem is TerminalAST) { return(CheckTokenOptions((TerminalAST)elem, ID, valueAST)); } if (elem.Type == ANTLRParser.ACTION) { return(false); } if (elem.Type == ANTLRParser.SEMPRED) { IToken optionID = ID.Token; string fileName = optionID.InputStream.SourceName; if (valueAST != null && !Grammar.semPredOptions.Contains(optionID.Text)) { g.tool.errMgr.GrammarError(ErrorType.ILLEGAL_OPTION, fileName, optionID, optionID.Text); return(false); } } return(false); }
public override void ElementOption(GrammarASTWithOptions t, GrammarAST ID, GrammarAST valueAST) { SetActionResolver(valueAST); }
public virtual string Text(GrammarAST t) { if (t == null) { return(""); } int tokenStartIndex = t.TokenStartIndex; int tokenStopIndex = t.TokenStopIndex; // ignore tokens from existing option subtrees like: // (ELEMENT_OPTIONS (= assoc right)) // // element options are added back according to the values in the map // returned by getOptions(). IntervalSet ignore = new IntervalSet(); IList <GrammarAST> optionsSubTrees = t.GetNodesWithType(ELEMENT_OPTIONS); foreach (GrammarAST sub in optionsSubTrees) { ignore.Add(sub.TokenStartIndex, sub.TokenStopIndex); } // Individual labels appear as RULE_REF or TOKEN_REF tokens in the tree, // but do not support the ELEMENT_OPTIONS syntax. Make sure to not try // and add the tokenIndex option when writing these tokens. IntervalSet noOptions = new IntervalSet(); IList <GrammarAST> labeledSubTrees = t.GetNodesWithType(new IntervalSet(ASSIGN, PLUS_ASSIGN)); foreach (GrammarAST sub in labeledSubTrees) { noOptions.Add(sub.GetChild(0).TokenStartIndex); } StringBuilder buf = new StringBuilder(); int i = tokenStartIndex; while (i <= tokenStopIndex) { if (ignore.Contains(i)) { i++; continue; } IToken tok = tokenStream.Get(i); // Compute/hold any element options StringBuilder elementOptions = new StringBuilder(); if (!noOptions.Contains(i)) { GrammarAST node = t.GetNodeWithTokenIndex(tok.TokenIndex); if (node != null && (tok.Type == TOKEN_REF || tok.Type == STRING_LITERAL || tok.Type == RULE_REF)) { elementOptions.Append("tokenIndex=").Append(tok.TokenIndex); } if (node is GrammarASTWithOptions) { GrammarASTWithOptions o = (GrammarASTWithOptions)node; foreach (KeyValuePair <string, GrammarAST> entry in o.GetOptions()) { if (elementOptions.Length > 0) { elementOptions.Append(','); } elementOptions.Append(entry.Key); elementOptions.Append('='); elementOptions.Append(entry.Value.Text); } } } buf.Append(tok.Text); // add actual text of the current token to the rewritten alternative i++; // move to the next token // Are there args on a rule? if (tok.Type == RULE_REF && i <= tokenStopIndex && tokenStream.Get(i).Type == ARG_ACTION) { buf.Append('[' + tokenStream.Get(i).Text + ']'); i++; } // now that we have the actual element, we can add the options. if (elementOptions.Length > 0) { buf.Append('<').Append(elementOptions).Append('>'); } } return(buf.ToString()); }
/** Build lexer grammar from combined grammar that looks like: * * (COMBINED_GRAMMAR A * (tokens { X (= Y 'y')) * (OPTIONS (= x 'y')) * (@ members {foo}) * (@ lexer header {package jj;}) * (RULES (RULE .+))) * * Move rules and actions to new tree, don't dup. Split AST apart. * We'll have this Grammar share token symbols later; don't generate * tokenVocab or tokens{} section. Copy over named actions. * * Side-effects: it removes children from GRAMMAR & RULES nodes * in combined AST. Anything cut out is dup'd before * adding to lexer to avoid "who's ur daddy" issues */ public virtual GrammarRootAST ExtractImplicitLexer(Grammar combinedGrammar) { GrammarRootAST combinedAST = combinedGrammar.ast; //tool.log("grammar", "before="+combinedAST.toStringTree()); GrammarASTAdaptor adaptor = new GrammarASTAdaptor(combinedAST.Token.InputStream); GrammarAST[] elements = combinedAST.GetChildrenAsArray(); // MAKE A GRAMMAR ROOT and ID string lexerName = combinedAST.GetChild(0).Text + "Lexer"; GrammarRootAST lexerAST = new GrammarRootAST(new CommonToken(ANTLRParser.GRAMMAR, "LEXER_GRAMMAR"), combinedGrammar.ast.tokenStream); lexerAST.grammarType = ANTLRParser.LEXER; lexerAST.Token.InputStream = combinedAST.Token.InputStream; lexerAST.AddChild((ITree)adaptor.Create(ANTLRParser.ID, lexerName)); // COPY OPTIONS GrammarAST optionsRoot = (GrammarAST)combinedAST.GetFirstChildWithType(ANTLRParser.OPTIONS); if (optionsRoot != null && optionsRoot.ChildCount != 0) { GrammarAST lexerOptionsRoot = (GrammarAST)adaptor.DupNode(optionsRoot); lexerAST.AddChild(lexerOptionsRoot); GrammarAST[] options = optionsRoot.GetChildrenAsArray(); foreach (GrammarAST o in options) { string optionName = o.GetChild(0).Text; if (Grammar.lexerOptions.Contains(optionName) && !Grammar.doNotCopyOptionsToLexer.Contains(optionName)) { GrammarAST optionTree = (GrammarAST)adaptor.DupTree(o); lexerOptionsRoot.AddChild(optionTree); lexerAST.SetOption(optionName, (GrammarAST)optionTree.GetChild(1)); } } } // COPY all named actions, but only move those with lexer:: scope IList <GrammarAST> actionsWeMoved = new List <GrammarAST>(); foreach (GrammarAST e in elements) { if (e.Type == ANTLRParser.AT) { lexerAST.AddChild((ITree)adaptor.DupTree(e)); if (e.GetChild(0).Text.Equals("lexer")) { actionsWeMoved.Add(e); } } } foreach (GrammarAST r in actionsWeMoved) { combinedAST.DeleteChild(r); } GrammarAST combinedRulesRoot = (GrammarAST)combinedAST.GetFirstChildWithType(ANTLRParser.RULES); if (combinedRulesRoot == null) { return(lexerAST); } // MOVE lexer rules GrammarAST lexerRulesRoot = (GrammarAST)adaptor.Create(ANTLRParser.RULES, "RULES"); lexerAST.AddChild(lexerRulesRoot); IList <GrammarAST> rulesWeMoved = new List <GrammarAST>(); GrammarASTWithOptions[] rules; if (combinedRulesRoot.ChildCount > 0) { rules = combinedRulesRoot.Children.Cast <GrammarASTWithOptions>().ToArray(); } else { rules = new GrammarASTWithOptions[0]; } foreach (GrammarASTWithOptions r in rules) { string ruleName = r.GetChild(0).Text; if (Grammar.IsTokenName(ruleName)) { lexerRulesRoot.AddChild((ITree)adaptor.DupTree(r)); rulesWeMoved.Add(r); } } foreach (GrammarAST r in rulesWeMoved) { combinedRulesRoot.DeleteChild(r); } // Will track 'if' from IF : 'if' ; rules to avoid defining new token for 'if' IList <System.Tuple <GrammarAST, GrammarAST> > litAliases = Grammar.GetStringLiteralAliasesFromLexerRules(lexerAST); ISet <string> stringLiterals = combinedGrammar.GetStringLiterals(); // add strings from combined grammar (and imported grammars) into lexer // put them first as they are keywords; must resolve ambigs to these rules // tool.log("grammar", "strings from parser: "+stringLiterals); int insertIndex = 0; foreach (string lit in stringLiterals) { // if lexer already has a rule for literal, continue if (litAliases != null) { foreach (System.Tuple <GrammarAST, GrammarAST> pair in litAliases) { GrammarAST litAST = pair.Item2; if (lit.Equals(litAST.Text)) { goto continueNextLit; } } } // create for each literal: (RULE <uniquename> (BLOCK (ALT <lit>)) string rname = combinedGrammar.GetStringLiteralLexerRuleName(lit); // can't use wizard; need special node types GrammarAST litRule = new RuleAST(ANTLRParser.RULE); BlockAST blk = new BlockAST(ANTLRParser.BLOCK); AltAST alt = new AltAST(ANTLRParser.ALT); TerminalAST slit = new TerminalAST(new CommonToken(ANTLRParser.STRING_LITERAL, lit)); alt.AddChild(slit); blk.AddChild(alt); CommonToken idToken = new CommonToken(ANTLRParser.TOKEN_REF, rname); litRule.AddChild(new TerminalAST(idToken)); litRule.AddChild(blk); lexerRulesRoot.InsertChild(insertIndex, litRule); // lexerRulesRoot.getChildren().add(0, litRule); lexerRulesRoot.FreshenParentAndChildIndexes(); // reset indexes and set litRule parent // next literal will be added after the one just added insertIndex++; continueNextLit: ; } // TODO: take out after stable if slow lexerAST.SanityCheckParentAndChildIndexes(); combinedAST.SanityCheckParentAndChildIndexes(); // tool.log("grammar", combinedAST.toTokenString()); combinedGrammar.tool.Log("grammar", "after extract implicit lexer =" + combinedAST.ToStringTree()); combinedGrammar.tool.Log("grammar", "lexer =" + lexerAST.ToStringTree()); if (lexerRulesRoot.ChildCount == 0) { return(null); } return(lexerAST); }