/** Build lexer grammar from combined grammar that looks like:
*
* (COMBINED_GRAMMAR A
* (tokens { X (= Y 'y'))
* (OPTIONS (= x 'y'))
* (@ members {foo})
* (@ lexer header {package jj;})
* (RULES (RULE .+)))
*
* Move rules and actions to new tree, don't dup. Split AST apart.
* We'll have this Grammar share token symbols later; don't generate
* tokenVocab or tokens{} section. Copy over named actions.
*
* Side-effects: it removes children from GRAMMAR & RULES nodes
* in combined AST. Anything cut out is dup'd before
* adding to lexer to avoid "who's ur daddy" issues
*/
public virtual GrammarRootAST ExtractImplicitLexer(Grammar combinedGrammar)
{
GrammarRootAST combinedAST = combinedGrammar.ast;
//tool.log("grammar", "before="+combinedAST.toStringTree());
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(combinedAST.Token.InputStream);
GrammarAST[] elements = combinedAST.GetChildrenAsArray();
// MAKE A GRAMMAR ROOT and ID
string lexerName = combinedAST.GetChild(0).Text + "Lexer";
GrammarRootAST lexerAST =
new GrammarRootAST(new CommonToken(ANTLRParser.GRAMMAR, "LEXER_GRAMMAR"), combinedGrammar.ast.tokenStream);
lexerAST.grammarType = ANTLRParser.LEXER;
lexerAST.Token.InputStream = combinedAST.Token.InputStream;
lexerAST.AddChild((ITree)adaptor.Create(ANTLRParser.ID, lexerName));
// COPY OPTIONS
GrammarAST optionsRoot =
(GrammarAST)combinedAST.GetFirstChildWithType(ANTLRParser.OPTIONS);
if (optionsRoot != null && optionsRoot.ChildCount != 0)
{
GrammarAST lexerOptionsRoot = (GrammarAST)adaptor.DupNode(optionsRoot);
lexerAST.AddChild(lexerOptionsRoot);
GrammarAST[] options = optionsRoot.GetChildrenAsArray();
foreach (GrammarAST o in options)
{
string optionName = o.GetChild(0).Text;
if (Grammar.lexerOptions.Contains(optionName) &&
!Grammar.doNotCopyOptionsToLexer.Contains(optionName))
{
GrammarAST optionTree = (GrammarAST)adaptor.DupTree(o);
lexerOptionsRoot.AddChild(optionTree);
lexerAST.SetOption(optionName, (GrammarAST)optionTree.GetChild(1));
}
}
}
// COPY all named actions, but only move those with lexer:: scope
IList <GrammarAST> actionsWeMoved = new List <GrammarAST>();
foreach (GrammarAST e in elements)
{
if (e.Type == ANTLRParser.AT)
{
lexerAST.AddChild((ITree)adaptor.DupTree(e));
if (e.GetChild(0).Text.Equals("lexer"))
{
actionsWeMoved.Add(e);
}
}
}
foreach (GrammarAST r in actionsWeMoved)
{
combinedAST.DeleteChild(r);
}
GrammarAST combinedRulesRoot =
(GrammarAST)combinedAST.GetFirstChildWithType(ANTLRParser.RULES);
if (combinedRulesRoot == null)
{
return(lexerAST);
}
// MOVE lexer rules
GrammarAST lexerRulesRoot = (GrammarAST)adaptor.Create(ANTLRParser.RULES, "RULES");
lexerAST.AddChild(lexerRulesRoot);
IList <GrammarAST> rulesWeMoved = new List <GrammarAST>();
GrammarASTWithOptions[] rules;
if (combinedRulesRoot.ChildCount > 0)
{
rules = combinedRulesRoot.Children.Cast <GrammarASTWithOptions>().ToArray();
}
else
{
rules = new GrammarASTWithOptions[0];
}
foreach (GrammarASTWithOptions r in rules)
{
string ruleName = r.GetChild(0).Text;
if (Grammar.IsTokenName(ruleName))
{
lexerRulesRoot.AddChild((ITree)adaptor.DupTree(r));
rulesWeMoved.Add(r);
}
}
foreach (GrammarAST r in rulesWeMoved)
{
combinedRulesRoot.DeleteChild(r);
}
// Will track 'if' from IF : 'if' ; rules to avoid defining new token for 'if'
IList <System.Tuple <GrammarAST, GrammarAST> > litAliases =
Grammar.GetStringLiteralAliasesFromLexerRules(lexerAST);
ISet <string> stringLiterals = combinedGrammar.GetStringLiterals();
// add strings from combined grammar (and imported grammars) into lexer
// put them first as they are keywords; must resolve ambigs to these rules
// tool.log("grammar", "strings from parser: "+stringLiterals);
int insertIndex = 0;
foreach (string lit in stringLiterals)
{
// if lexer already has a rule for literal, continue
if (litAliases != null)
{
foreach (System.Tuple <GrammarAST, GrammarAST> pair in litAliases)
{
GrammarAST litAST = pair.Item2;
if (lit.Equals(litAST.Text))
{
goto continueNextLit;
}
}
}
// create for each literal: (RULE <uniquename> (BLOCK (ALT <lit>))
string rname = combinedGrammar.GetStringLiteralLexerRuleName(lit);
// can't use wizard; need special node types
GrammarAST litRule = new RuleAST(ANTLRParser.RULE);
BlockAST blk = new BlockAST(ANTLRParser.BLOCK);
AltAST alt = new AltAST(ANTLRParser.ALT);
TerminalAST slit = new TerminalAST(new CommonToken(ANTLRParser.STRING_LITERAL, lit));
alt.AddChild(slit);
blk.AddChild(alt);
CommonToken idToken = new CommonToken(ANTLRParser.TOKEN_REF, rname);
litRule.AddChild(new TerminalAST(idToken));
litRule.AddChild(blk);
lexerRulesRoot.InsertChild(insertIndex, litRule);
// lexerRulesRoot.getChildren().add(0, litRule);
lexerRulesRoot.FreshenParentAndChildIndexes(); // reset indexes and set litRule parent
// next literal will be added after the one just added
insertIndex++;
continueNextLit:
;
}
// TODO: take out after stable if slow
lexerAST.SanityCheckParentAndChildIndexes();
combinedAST.SanityCheckParentAndChildIndexes();
// tool.log("grammar", combinedAST.toTokenString());
combinedGrammar.tool.Log("grammar", "after extract implicit lexer =" + combinedAST.ToStringTree());
combinedGrammar.tool.Log("grammar", "lexer =" + lexerAST.ToStringTree());
if (lexerRulesRoot.ChildCount == 0)
{
return(null);
}
return(lexerAST);
}
/** Merge all the rules, token definitions, and named actions from
* imported grammars into the root grammar tree. Perform:
*
* (tokens { X (= Y 'y')) + (tokens { Z ) -> (tokens { X (= Y 'y') Z)
*
* (@ members {foo}) + (@ members {bar}) -> (@ members {foobar})
*
* (RULES (RULE x y)) + (RULES (RULE z)) -> (RULES (RULE x y z))
*
* Rules in root prevent same rule from being appended to RULES node.
*
* The goal is a complete combined grammar so we can ignore subordinate
* grammars.
*/
public virtual void IntegrateImportedGrammars(Grammar rootGrammar)
{
IList <Grammar> imports = rootGrammar.GetAllImportedGrammars();
if (imports == null)
{
return;
}
GrammarAST root = rootGrammar.ast;
GrammarAST id = (GrammarAST)root.GetChild(0);
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(id.Token.InputStream);
GrammarAST tokensRoot = (GrammarAST)root.GetFirstChildWithType(ANTLRParser.TOKENS_SPEC);
IList <GrammarAST> actionRoots = root.GetNodesWithType(ANTLRParser.AT);
// Compute list of rules in root grammar and ensure we have a RULES node
GrammarAST RULES = (GrammarAST)root.GetFirstChildWithType(ANTLRParser.RULES);
ISet <string> rootRuleNames = new HashSet <string>();
// make list of rules we have in root grammar
IList <GrammarAST> rootRules = RULES.GetNodesWithType(ANTLRParser.RULE);
foreach (GrammarAST r in rootRules)
{
rootRuleNames.Add(r.GetChild(0).Text);
}
foreach (Grammar imp in imports)
{
// COPY TOKENS
GrammarAST imp_tokensRoot = (GrammarAST)imp.ast.GetFirstChildWithType(ANTLRParser.TOKENS_SPEC);
if (imp_tokensRoot != null)
{
rootGrammar.tool.Log("grammar", "imported tokens: " + imp_tokensRoot.Children);
if (tokensRoot == null)
{
tokensRoot = (GrammarAST)adaptor.Create(ANTLRParser.TOKENS_SPEC, "TOKENS");
tokensRoot.g = rootGrammar;
root.InsertChild(1, tokensRoot); // ^(GRAMMAR ID TOKENS...)
}
tokensRoot.AddChildren(imp_tokensRoot.Children);
}
IList <GrammarAST> all_actionRoots = new List <GrammarAST>();
IList <GrammarAST> imp_actionRoots = imp.ast.GetAllChildrenWithType(ANTLRParser.AT);
if (actionRoots != null)
{
foreach (var actionRoot in actionRoots)
{
all_actionRoots.Add(actionRoot);
}
}
foreach (var actionRoot in imp_actionRoots)
{
all_actionRoots.Add(actionRoot);
}
// COPY ACTIONS
if (imp_actionRoots != null)
{
IDictionary <System.Tuple <string, string>, GrammarAST> namedActions =
new Dictionary <System.Tuple <string, string>, GrammarAST>();
rootGrammar.tool.Log("grammar", "imported actions: " + imp_actionRoots);
foreach (GrammarAST at in all_actionRoots)
{
string scopeName = rootGrammar.GetDefaultActionScope();
GrammarAST scope, name, action;
if (at.ChildCount > 2)
{ // must have a scope
scope = (GrammarAST)at.GetChild(0);
scopeName = scope.Text;
name = (GrammarAST)at.GetChild(1);
action = (GrammarAST)at.GetChild(2);
}
else
{
name = (GrammarAST)at.GetChild(0);
action = (GrammarAST)at.GetChild(1);
}
GrammarAST prevAction;
if (!namedActions.TryGetValue(Tuple.Create(scopeName, name.Text), out prevAction) || prevAction == null)
{
namedActions[Tuple.Create(scopeName, name.Text)] = action;
}
else
{
if (prevAction.g == at.g)
{
rootGrammar.tool.errMgr.GrammarError(ErrorType.ACTION_REDEFINITION,
at.g.fileName, name.Token, name.Text);
}
else
{
string s1 = prevAction.Text;
s1 = s1.Substring(1, s1.Length - 2);
string s2 = action.Text;
s2 = s2.Substring(1, s2.Length - 2);
string combinedAction = "{" + s1 + '\n' + s2 + "}";
prevAction.Token.Text = combinedAction;
}
}
}
// at this point, we have complete list of combined actions,
// some of which are already living in root grammar.
// Merge in any actions not in root grammar into root's tree.
foreach (string scopeName in namedActions.Keys.Select(i => i.Item1).Distinct())
{
foreach (string name in namedActions.Keys.Where(i => i.Item1 == scopeName).Select(i => i.Item2))
{
GrammarAST action = namedActions[Tuple.Create(scopeName, name)];
rootGrammar.tool.Log("grammar", action.g.name + " " + scopeName + ":" + name + "=" + action.Text);
if (action.g != rootGrammar)
{
root.InsertChild(1, action.Parent);
}
}
}
}
// COPY RULES
IList <GrammarAST> rules = imp.ast.GetNodesWithType(ANTLRParser.RULE);
if (rules != null)
{
foreach (GrammarAST r in rules)
{
rootGrammar.tool.Log("grammar", "imported rule: " + r.ToStringTree());
string name = r.GetChild(0).Text;
bool rootAlreadyHasRule = rootRuleNames.Contains(name);
if (!rootAlreadyHasRule)
{
RULES.AddChild(r); // merge in if not overridden
rootRuleNames.Add(name);
}
}
}
GrammarAST optionsRoot = (GrammarAST)imp.ast.GetFirstChildWithType(ANTLRParser.OPTIONS);
if (optionsRoot != null)
{
// suppress the warning if the options match the options specified
// in the root grammar
// https://github.com/antlr/antlr4/issues/707
bool hasNewOption = false;
foreach (KeyValuePair <string, GrammarAST> option in imp.ast.GetOptions())
{
string importOption = imp.ast.GetOptionString(option.Key);
if (importOption == null)
{
continue;
}
string rootOption = rootGrammar.ast.GetOptionString(option.Key);
if (!importOption.Equals(rootOption))
{
hasNewOption = true;
break;
}
}
if (hasNewOption)
{
rootGrammar.tool.errMgr.GrammarError(ErrorType.OPTIONS_IN_DELEGATE,
optionsRoot.g.fileName, optionsRoot.Token, imp.name);
}
}
}
rootGrammar.tool.Log("grammar", "Grammar: " + rootGrammar.ast.ToStringTree());
}
