public static void AugmentTokensWithOriginalPosition(Grammar g, GrammarAST tree)
{
if (tree == null)
{
return;
}
IList <GrammarAST> optionsSubTrees = tree.GetNodesWithType(ANTLRParser.ELEMENT_OPTIONS);
for (int i = 0; i < optionsSubTrees.Count; i++)
{
GrammarAST t = optionsSubTrees[i];
CommonTree elWithOpt = (CommonTree)t.Parent;
if (elWithOpt is GrammarASTWithOptions)
{
IDictionary <string, GrammarAST> options = ((GrammarASTWithOptions)elWithOpt).GetOptions();
if (options.ContainsKey(LeftRecursiveRuleTransformer.TOKENINDEX_OPTION_NAME))
{
GrammarToken newTok = new GrammarToken(g, elWithOpt.Token);
newTok.originalTokenIndex = int.Parse(options[LeftRecursiveRuleTransformer.TOKENINDEX_OPTION_NAME].Text);
elWithOpt.Token = newTok;
GrammarAST originalNode = g.ast.GetNodeWithTokenIndex(newTok.TokenIndex);
if (originalNode != null)
{
// update the AST node start/stop index to match the values
// of the corresponding node in the original parse tree.
elWithOpt.TokenStartIndex = originalNode.TokenStartIndex;
elWithOpt.TokenStopIndex = originalNode.TokenStopIndex;
}
else
{
// the original AST node could not be located by index;
// make sure to assign valid values for the start/stop
// index so toTokenString will not throw exceptions.
elWithOpt.TokenStartIndex = newTok.TokenIndex;
elWithOpt.TokenStopIndex = newTok.TokenIndex;
}
}
}
}
}
public virtual string Text(GrammarAST t)
{
if (t == null)
{
return("");
}
int tokenStartIndex = t.TokenStartIndex;
int tokenStopIndex = t.TokenStopIndex;
// ignore tokens from existing option subtrees like:
// (ELEMENT_OPTIONS (= assoc right))
//
// element options are added back according to the values in the map
// returned by getOptions().
IntervalSet ignore = new IntervalSet();
IList <GrammarAST> optionsSubTrees = t.GetNodesWithType(ELEMENT_OPTIONS);
foreach (GrammarAST sub in optionsSubTrees)
{
ignore.Add(sub.TokenStartIndex, sub.TokenStopIndex);
}
// Individual labels appear as RULE_REF or TOKEN_REF tokens in the tree,
// but do not support the ELEMENT_OPTIONS syntax. Make sure to not try
// and add the tokenIndex option when writing these tokens.
IntervalSet noOptions = new IntervalSet();
IList <GrammarAST> labeledSubTrees = t.GetNodesWithType(new IntervalSet(ASSIGN, PLUS_ASSIGN));
foreach (GrammarAST sub in labeledSubTrees)
{
noOptions.Add(sub.GetChild(0).TokenStartIndex);
}
StringBuilder buf = new StringBuilder();
int i = tokenStartIndex;
while (i <= tokenStopIndex)
{
if (ignore.Contains(i))
{
i++;
continue;
}
IToken tok = tokenStream.Get(i);
// Compute/hold any element options
StringBuilder elementOptions = new StringBuilder();
if (!noOptions.Contains(i))
{
GrammarAST node = t.GetNodeWithTokenIndex(tok.TokenIndex);
if (node != null &&
(tok.Type == TOKEN_REF ||
tok.Type == STRING_LITERAL ||
tok.Type == RULE_REF))
{
elementOptions.Append("tokenIndex=").Append(tok.TokenIndex);
}
if (node is GrammarASTWithOptions)
{
GrammarASTWithOptions o = (GrammarASTWithOptions)node;
foreach (KeyValuePair <string, GrammarAST> entry in o.GetOptions())
{
if (elementOptions.Length > 0)
{
elementOptions.Append(',');
}
elementOptions.Append(entry.Key);
elementOptions.Append('=');
elementOptions.Append(entry.Value.Text);
}
}
}
buf.Append(tok.Text); // add actual text of the current token to the rewritten alternative
i++; // move to the next token
// Are there args on a rule?
if (tok.Type == RULE_REF && i <= tokenStopIndex && tokenStream.Get(i).Type == ARG_ACTION)
{
buf.Append('[' + tokenStream.Get(i).Text + ']');
i++;
}
// now that we have the actual element, we can add the options.
if (elementOptions.Length > 0)
{
buf.Append('<').Append(elementOptions).Append('>');
}
}
return(buf.ToString());
}
/** Merge all the rules, token definitions, and named actions from
* imported grammars into the root grammar tree. Perform:
*
* (tokens { X (= Y 'y')) + (tokens { Z ) -> (tokens { X (= Y 'y') Z)
*
* (@ members {foo}) + (@ members {bar}) -> (@ members {foobar})
*
* (RULES (RULE x y)) + (RULES (RULE z)) -> (RULES (RULE x y z))
*
* Rules in root prevent same rule from being appended to RULES node.
*
* The goal is a complete combined grammar so we can ignore subordinate
* grammars.
*/
public virtual void IntegrateImportedGrammars(Grammar rootGrammar)
{
IList <Grammar> imports = rootGrammar.GetAllImportedGrammars();
if (imports == null)
{
return;
}
GrammarAST root = rootGrammar.ast;
GrammarAST id = (GrammarAST)root.GetChild(0);
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(id.Token.InputStream);
GrammarAST tokensRoot = (GrammarAST)root.GetFirstChildWithType(ANTLRParser.TOKENS_SPEC);
IList <GrammarAST> actionRoots = root.GetNodesWithType(ANTLRParser.AT);
// Compute list of rules in root grammar and ensure we have a RULES node
GrammarAST RULES = (GrammarAST)root.GetFirstChildWithType(ANTLRParser.RULES);
ISet <string> rootRuleNames = new HashSet <string>();
// make list of rules we have in root grammar
IList <GrammarAST> rootRules = RULES.GetNodesWithType(ANTLRParser.RULE);
foreach (GrammarAST r in rootRules)
{
rootRuleNames.Add(r.GetChild(0).Text);
}
foreach (Grammar imp in imports)
{
// COPY TOKENS
GrammarAST imp_tokensRoot = (GrammarAST)imp.ast.GetFirstChildWithType(ANTLRParser.TOKENS_SPEC);
if (imp_tokensRoot != null)
{
rootGrammar.tool.Log("grammar", "imported tokens: " + imp_tokensRoot.Children);
if (tokensRoot == null)
{
tokensRoot = (GrammarAST)adaptor.Create(ANTLRParser.TOKENS_SPEC, "TOKENS");
tokensRoot.g = rootGrammar;
root.InsertChild(1, tokensRoot); // ^(GRAMMAR ID TOKENS...)
}
tokensRoot.AddChildren(imp_tokensRoot.Children);
}
IList <GrammarAST> all_actionRoots = new List <GrammarAST>();
IList <GrammarAST> imp_actionRoots = imp.ast.GetAllChildrenWithType(ANTLRParser.AT);
if (actionRoots != null)
{
foreach (var actionRoot in actionRoots)
{
all_actionRoots.Add(actionRoot);
}
}
foreach (var actionRoot in imp_actionRoots)
{
all_actionRoots.Add(actionRoot);
}
// COPY ACTIONS
if (imp_actionRoots != null)
{
IDictionary <System.Tuple <string, string>, GrammarAST> namedActions =
new Dictionary <System.Tuple <string, string>, GrammarAST>();
rootGrammar.tool.Log("grammar", "imported actions: " + imp_actionRoots);
foreach (GrammarAST at in all_actionRoots)
{
string scopeName = rootGrammar.GetDefaultActionScope();
GrammarAST scope, name, action;
if (at.ChildCount > 2)
{ // must have a scope
scope = (GrammarAST)at.GetChild(0);
scopeName = scope.Text;
name = (GrammarAST)at.GetChild(1);
action = (GrammarAST)at.GetChild(2);
}
else
{
name = (GrammarAST)at.GetChild(0);
action = (GrammarAST)at.GetChild(1);
}
GrammarAST prevAction;
if (!namedActions.TryGetValue(Tuple.Create(scopeName, name.Text), out prevAction) || prevAction == null)
{
namedActions[Tuple.Create(scopeName, name.Text)] = action;
}
else
{
if (prevAction.g == at.g)
{
rootGrammar.tool.errMgr.GrammarError(ErrorType.ACTION_REDEFINITION,
at.g.fileName, name.Token, name.Text);
}
else
{
string s1 = prevAction.Text;
s1 = s1.Substring(1, s1.Length - 2);
string s2 = action.Text;
s2 = s2.Substring(1, s2.Length - 2);
string combinedAction = "{" + s1 + '\n' + s2 + "}";
prevAction.Token.Text = combinedAction;
}
}
}
// at this point, we have complete list of combined actions,
// some of which are already living in root grammar.
// Merge in any actions not in root grammar into root's tree.
foreach (string scopeName in namedActions.Keys.Select(i => i.Item1).Distinct())
{
foreach (string name in namedActions.Keys.Where(i => i.Item1 == scopeName).Select(i => i.Item2))
{
GrammarAST action = namedActions[Tuple.Create(scopeName, name)];
rootGrammar.tool.Log("grammar", action.g.name + " " + scopeName + ":" + name + "=" + action.Text);
if (action.g != rootGrammar)
{
root.InsertChild(1, action.Parent);
}
}
}
}
// COPY RULES
IList <GrammarAST> rules = imp.ast.GetNodesWithType(ANTLRParser.RULE);
if (rules != null)
{
foreach (GrammarAST r in rules)
{
rootGrammar.tool.Log("grammar", "imported rule: " + r.ToStringTree());
string name = r.GetChild(0).Text;
bool rootAlreadyHasRule = rootRuleNames.Contains(name);
if (!rootAlreadyHasRule)
{
RULES.AddChild(r); // merge in if not overridden
rootRuleNames.Add(name);
}
}
}
GrammarAST optionsRoot = (GrammarAST)imp.ast.GetFirstChildWithType(ANTLRParser.OPTIONS);
if (optionsRoot != null)
{
// suppress the warning if the options match the options specified
// in the root grammar
// https://github.com/antlr/antlr4/issues/707
bool hasNewOption = false;
foreach (KeyValuePair <string, GrammarAST> option in imp.ast.GetOptions())
{
string importOption = imp.ast.GetOptionString(option.Key);
if (importOption == null)
{
continue;
}
string rootOption = rootGrammar.ast.GetOptionString(option.Key);
if (!importOption.Equals(rootOption))
{
hasNewOption = true;
break;
}
}
if (hasNewOption)
{
rootGrammar.tool.errMgr.GrammarError(ErrorType.OPTIONS_IN_DELEGATE,
optionsRoot.g.fileName, optionsRoot.Token, imp.name);
}
}
}
rootGrammar.tool.Log("grammar", "Grammar: " + rootGrammar.ast.ToStringTree());
}