protected StringTemplate GetRuleElementST(string name,
string ruleTargetName,
GrammarAST elementAST,
GrammarAST ast_suffix,
string label)
{
string suffix = GetSTSuffix(elementAST, ast_suffix, label);
name += suffix;
// if we're building trees and there is no label, gen a label
// unless we're in a synpred rule.
Rule r = grammar.GetRule(currentRuleName);
if ((grammar.BuildAST || suffix.Length > 0) && label == null &&
(r == null || !r.isSynPred))
{
// we will need a label to do the AST or tracking, make one
label = generator.CreateUniqueLabel(ruleTargetName);
CommonToken labelTok = new CommonToken(ANTLRParser.ID, label);
grammar.DefineRuleRefLabel(currentRuleName, labelTok, elementAST);
}
StringTemplate elementST = templates.GetInstanceOf(name);
if (label != null)
{
elementST.SetAttribute("label", label);
}
return(elementST);
}
/** For references to tokens rather than by label such as $ID, we
* need to get the existing label for the ID ref or create a new
* one.
*/
public virtual string GetElementLabel(string refdSymbol,
int outerAltNum,
CodeGenerator generator)
{
GrammarAST uniqueRefAST;
if (Grammar.type != GrammarType.Lexer &&
Rule.GetRuleType(refdSymbol) == RuleType.Lexer)
{
// symbol is a token
IList <GrammarAST> tokenRefs = GetTokenRefsInAlt(refdSymbol, outerAltNum);
uniqueRefAST = tokenRefs[0];
}
else
{
// symbol is a rule
IList <GrammarAST> ruleRefs = GetRuleRefsInAlt(refdSymbol, outerAltNum);
uniqueRefAST = ruleRefs[0];
}
if (uniqueRefAST.code == null)
{
// no code? must not have gen'd yet; forward ref
return(null);
}
string labelName = null;
string existingLabelName =
(string)uniqueRefAST.code.GetAttribute("label");
// reuse any label or list label if it exists
if (existingLabelName != null)
{
labelName = existingLabelName;
}
else
{
// else create new label
labelName = generator.CreateUniqueLabel(refdSymbol);
CommonToken label = new CommonToken(ANTLRParser.ID, labelName);
if (Grammar.type != GrammarType.Lexer &&
Rule.GetRuleType(refdSymbol) == Tool.RuleType.Lexer)
{
Grammar.DefineTokenRefLabel(Name, label, uniqueRefAST);
}
else
{
Grammar.DefineRuleRefLabel(Name, label, uniqueRefAST);
}
uniqueRefAST.code.SetAttribute("label", labelName);
}
return(labelName);
}
protected StringTemplate GetTokenElementST(string name,
string elementName,
GrammarAST elementAST,
GrammarAST ast_suffix,
string label)
{
bool tryUnchecked = false;
if (name == "matchSet" && !string.IsNullOrEmpty(elementAST.enclosingRuleName) && Rule.GetRuleType(elementAST.enclosingRuleName) == RuleType.Lexer)
{
if ((elementAST.Parent.Type == ANTLRLexer.ALT && elementAST.Parent.Parent.Parent.Type == RULE && elementAST.Parent.Parent.ChildCount == 2) ||
(elementAST.Parent.Type == ANTLRLexer.NOT && elementAST.Parent.Parent.Parent.Parent.Type == RULE && elementAST.Parent.Parent.Parent.ChildCount == 2))
{
// single alt at the start of the rule needs to be checked
}
else
{
tryUnchecked = true;
}
}
string suffix = GetSTSuffix(elementAST, ast_suffix, label);
// if we're building trees and there is no label, gen a label
// unless we're in a synpred rule.
Rule r = grammar.GetRule(currentRuleName);
if ((grammar.BuildAST || suffix.Length > 0) && label == null &&
(r == null || !r.isSynPred))
{
label = generator.CreateUniqueLabel(elementName);
CommonToken labelTok = new CommonToken(ANTLRParser.ID, label);
grammar.DefineTokenRefLabel(currentRuleName, labelTok, elementAST);
}
StringTemplate elementST = null;
if (tryUnchecked && templates.IsDefined(name + "Unchecked" + suffix))
{
elementST = templates.GetInstanceOf(name + "Unchecked" + suffix);
}
if (elementST == null)
{
elementST = templates.GetInstanceOf(name + suffix);
}
if (label != null)
{
elementST.SetAttribute("label", label);
}
return(elementST);
}
public object ParseNode()
{
// "%label:" prefix
string label = null;
if (ttype == TreePatternLexer.PERCENT)
{
ttype = tokenizer.NextToken();
if (ttype != TreePatternLexer.ID)
{
return null;
}
label = tokenizer.sval.ToString();
ttype = tokenizer.NextToken();
if (ttype != TreePatternLexer.COLON)
{
return null;
}
ttype = tokenizer.NextToken(); // move to ID following colon
}
// Wildcard?
if (ttype == TreePatternLexer.DOT)
{
ttype = tokenizer.NextToken();
IToken wildcardPayload = new CommonToken(0, ".");
TreeWizard.TreePattern node = new TreeWizard.WildcardTreePattern(wildcardPayload);
if (label != null)
{
node.label = label;
}
return node;
}
// "ID" or "ID[arg]"
if (ttype != TreePatternLexer.ID)
{
return null;
}
string tokenName = tokenizer.sval.ToString();
ttype = tokenizer.NextToken();
if (tokenName.Equals("nil"))
{
return adaptor.GetNilNode();
}
string text = tokenName;
// check for arg
string arg = null;
if (ttype == TreePatternLexer.ARG)
{
arg = tokenizer.sval.ToString();
text = arg;
ttype = tokenizer.NextToken();
}
// create node
int treeNodeType = wizard.GetTokenType(tokenName);
if (treeNodeType == Token.INVALID_TOKEN_TYPE)
{
return null;
}
object nodeObj;
nodeObj = adaptor.Create(treeNodeType, text);
if ((label != null) && (nodeObj.GetType() == typeof(TreeWizard.TreePattern)))
{
((TreeWizard.TreePattern)nodeObj).label = label;
}
if ((arg != null) && (nodeObj.GetType() == typeof(TreeWizard.TreePattern)))
{
((TreeWizard.TreePattern)nodeObj).hasTextArg = true;
}
return nodeObj;
}
protected StringTemplate GetTokenElementST( string name,
string elementName,
GrammarAST elementAST,
GrammarAST ast_suffix,
string label )
{
bool tryUnchecked = false;
if (name == "matchSet" && !string.IsNullOrEmpty(elementAST.enclosingRuleName) && Rule.GetRuleType(elementAST.enclosingRuleName) == RuleType.Lexer)
{
if ( ( elementAST.Parent.Type == ANTLRLexer.ALT && elementAST.Parent.Parent.Parent.Type == RULE && elementAST.Parent.Parent.ChildCount == 2 )
|| ( elementAST.Parent.Type == ANTLRLexer.NOT && elementAST.Parent.Parent.Parent.Parent.Type == RULE && elementAST.Parent.Parent.Parent.ChildCount == 2 ) )
{
// single alt at the start of the rule needs to be checked
}
else
{
tryUnchecked = true;
}
}
string suffix = GetSTSuffix( elementAST, ast_suffix, label );
// if we're building trees and there is no label, gen a label
// unless we're in a synpred rule.
Rule r = grammar.GetRule( currentRuleName );
if ( ( grammar.BuildAST || suffix.Length > 0 ) && label == null &&
( r == null || !r.isSynPred ) )
{
label = generator.CreateUniqueLabel( elementName );
CommonToken labelTok = new CommonToken( ANTLRParser.ID, label );
grammar.DefineTokenRefLabel( currentRuleName, labelTok, elementAST );
}
StringTemplate elementST = null;
if ( tryUnchecked && templates.IsDefined( name + "Unchecked" + suffix ) )
elementST = templates.GetInstanceOf( name + "Unchecked" + suffix );
if ( elementST == null )
elementST = templates.GetInstanceOf( name + suffix );
if ( label != null )
{
elementST.SetAttribute( "label", label );
}
return elementST;
}
protected StringTemplate GetRuleElementST( string name,
string ruleTargetName,
GrammarAST elementAST,
GrammarAST ast_suffix,
string label )
{
string suffix = GetSTSuffix( elementAST, ast_suffix, label );
name += suffix;
// if we're building trees and there is no label, gen a label
// unless we're in a synpred rule.
Rule r = grammar.GetRule( currentRuleName );
if ( ( grammar.BuildAST || suffix.Length > 0 ) && label == null &&
( r == null || !r.isSynPred ) )
{
// we will need a label to do the AST or tracking, make one
label = generator.CreateUniqueLabel( ruleTargetName );
CommonToken labelTok = new CommonToken( ANTLRParser.ID, label );
grammar.DefineRuleRefLabel( currentRuleName, labelTok, elementAST );
}
StringTemplate elementST = templates.GetInstanceOf( name );
if ( label != null )
{
elementST.SetAttribute( "label", label );
}
return elementST;
}
public virtual void Initialize(int i, string s)
{
Token = new CommonToken(i, s);
Token.TokenIndex = -1;
}
/** For references to tokens rather than by label such as $ID, we
* need to get the existing label for the ID ref or create a new
* one.
*/
public virtual string GetElementLabel( string refdSymbol,
int outerAltNum,
CodeGenerator generator )
{
GrammarAST uniqueRefAST;
if ( grammar.type != GrammarType.Lexer &&
char.IsUpper( refdSymbol[0] ) )
{
// symbol is a token
IList tokenRefs = GetTokenRefsInAlt( refdSymbol, outerAltNum );
uniqueRefAST = (GrammarAST)tokenRefs[0];
}
else
{
// symbol is a rule
IList ruleRefs = GetRuleRefsInAlt( refdSymbol, outerAltNum );
uniqueRefAST = (GrammarAST)ruleRefs[0];
}
if ( uniqueRefAST.code == null )
{
// no code? must not have gen'd yet; forward ref
return null;
}
string labelName = null;
string existingLabelName =
(string)uniqueRefAST.code.GetAttribute( "label" );
// reuse any label or list label if it exists
if ( existingLabelName != null )
{
labelName = existingLabelName;
}
else
{
// else create new label
labelName = generator.CreateUniqueLabel( refdSymbol );
CommonToken label = new CommonToken( ANTLRParser.ID, labelName );
if ( grammar.type != GrammarType.Lexer &&
char.IsUpper( refdSymbol[0] ) )
{
grammar.DefineTokenRefLabel( Name, label, uniqueRefAST );
}
else
{
grammar.DefineRuleRefLabel( Name, label, uniqueRefAST );
}
uniqueRefAST.code.SetAttribute( "label", labelName );
}
return labelName;
}
public virtual void Initialize( int i, string s )
{
token = new CommonToken( i, s );
}
public object ParseNode()
{
// "%label:" prefix
string label = null;
if (ttype == TreePatternLexer.PERCENT)
{
ttype = tokenizer.NextToken();
if (ttype != TreePatternLexer.ID)
{
return(null);
}
label = tokenizer.sval.ToString();
ttype = tokenizer.NextToken();
if (ttype != TreePatternLexer.COLON)
{
return(null);
}
ttype = tokenizer.NextToken(); // move to ID following colon
}
// Wildcard?
if (ttype == TreePatternLexer.DOT)
{
ttype = tokenizer.NextToken();
IToken wildcardPayload = new CommonToken(0, ".");
TreeWizard.TreePattern node = new TreeWizard.WildcardTreePattern(wildcardPayload);
if (label != null)
{
node.label = label;
}
return(node);
}
// "ID" or "ID[arg]"
if (ttype != TreePatternLexer.ID)
{
return(null);
}
string tokenName = tokenizer.sval.ToString();
ttype = tokenizer.NextToken();
if (tokenName.Equals("nil"))
{
return(adaptor.GetNilNode());
}
string text = tokenName;
// check for arg
string arg = null;
if (ttype == TreePatternLexer.ARG)
{
arg = tokenizer.sval.ToString();
text = arg;
ttype = tokenizer.NextToken();
}
// create node
int treeNodeType = wizard.GetTokenType(tokenName);
if (treeNodeType == Token.INVALID_TOKEN_TYPE)
{
return(null);
}
object nodeObj;
nodeObj = adaptor.Create(treeNodeType, text);
if ((label != null) && (nodeObj.GetType() == typeof(TreeWizard.TreePattern)))
{
((TreeWizard.TreePattern)nodeObj).label = label;
}
if ((arg != null) && (nodeObj.GetType() == typeof(TreeWizard.TreePattern)))
{
((TreeWizard.TreePattern)nodeObj).hasTextArg = true;
}
return(nodeObj);
}
public V( int ttype, int x )
{
this.x = x;
Token = new CommonToken( ttype );
}
/** Translate an action like [3,"foo",a[3]] and return a List of the
* translated actions. Because actions are themselves translated to a list
* of chunks, must cat together into a StringTemplate>. Don't translate
* to strings early as we need to eval templates in context.
*/
public virtual List<StringTemplate> TranslateArgAction( string ruleName,
GrammarAST actionTree )
{
string actionText = actionTree.Token.Text;
List<string> args = GetListOfArgumentsFromAction( actionText, ',' );
List<StringTemplate> translatedArgs = new List<StringTemplate>();
foreach ( string arg in args )
{
if ( arg != null )
{
IToken actionToken =
new CommonToken( ANTLRParser.ACTION, arg );
ActionTranslator translator =
new ActionTranslator( this, ruleName,
actionToken,
actionTree.outerAltNum );
IList chunks = translator.TranslateToChunks();
chunks = target.PostProcessAction( chunks, actionToken );
StringTemplate catST = new StringTemplate( templates, "<chunks>" );
catST.SetAttribute( "chunks", chunks );
templates.CreateStringTemplate();
translatedArgs.Add( catST );
}
}
if ( translatedArgs.Count == 0 )
{
return null;
}
return translatedArgs;
}
/** Build lexer grammar from combined grammar that looks like:
*
* (COMBINED_GRAMMAR A
* (tokens { X (= Y 'y'))
* (OPTIONS (= x 'y'))
* (@ members {foo})
* (@ lexer header {package jj;})
* (RULES (RULE .+)))
*
* Move rules and actions to new tree, don't dup. Split AST apart.
* We'll have this Grammar share token symbols later; don't generate
* tokenVocab or tokens{} section. Copy over named actions.
*
* Side-effects: it removes children from GRAMMAR & RULES nodes
* in combined AST. Anything cut out is dup'd before
* adding to lexer to avoid "who's ur daddy" issues
*/
public virtual GrammarRootAST ExtractImplicitLexer(Grammar combinedGrammar)
{
GrammarRootAST combinedAST = combinedGrammar.ast;
//tool.log("grammar", "before="+combinedAST.toStringTree());
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(combinedAST.Token.InputStream);
GrammarAST[] elements = combinedAST.GetChildrenAsArray();
// MAKE A GRAMMAR ROOT and ID
string lexerName = combinedAST.GetChild(0).Text + "Lexer";
GrammarRootAST lexerAST =
new GrammarRootAST(new CommonToken(ANTLRParser.GRAMMAR, "LEXER_GRAMMAR"), combinedGrammar.ast.tokenStream);
lexerAST.grammarType = ANTLRParser.LEXER;
lexerAST.Token.InputStream = combinedAST.Token.InputStream;
lexerAST.AddChild((ITree)adaptor.Create(ANTLRParser.ID, lexerName));
// COPY OPTIONS
GrammarAST optionsRoot =
(GrammarAST)combinedAST.GetFirstChildWithType(ANTLRParser.OPTIONS);
if (optionsRoot != null && optionsRoot.ChildCount != 0)
{
GrammarAST lexerOptionsRoot = (GrammarAST)adaptor.DupNode(optionsRoot);
lexerAST.AddChild(lexerOptionsRoot);
GrammarAST[] options = optionsRoot.GetChildrenAsArray();
foreach (GrammarAST o in options)
{
string optionName = o.GetChild(0).Text;
if (Grammar.lexerOptions.Contains(optionName) &&
!Grammar.doNotCopyOptionsToLexer.Contains(optionName))
{
GrammarAST optionTree = (GrammarAST)adaptor.DupTree(o);
lexerOptionsRoot.AddChild(optionTree);
lexerAST.SetOption(optionName, (GrammarAST)optionTree.GetChild(1));
}
}
}
// COPY all named actions, but only move those with lexer:: scope
IList <GrammarAST> actionsWeMoved = new List <GrammarAST>();
foreach (GrammarAST e in elements)
{
if (e.Type == ANTLRParser.AT)
{
lexerAST.AddChild((ITree)adaptor.DupTree(e));
if (e.GetChild(0).Text.Equals("lexer"))
{
actionsWeMoved.Add(e);
}
}
}
foreach (GrammarAST r in actionsWeMoved)
{
combinedAST.DeleteChild(r);
}
GrammarAST combinedRulesRoot =
(GrammarAST)combinedAST.GetFirstChildWithType(ANTLRParser.RULES);
if (combinedRulesRoot == null)
{
return(lexerAST);
}
// MOVE lexer rules
GrammarAST lexerRulesRoot = (GrammarAST)adaptor.Create(ANTLRParser.RULES, "RULES");
lexerAST.AddChild(lexerRulesRoot);
IList <GrammarAST> rulesWeMoved = new List <GrammarAST>();
GrammarASTWithOptions[] rules;
if (combinedRulesRoot.ChildCount > 0)
{
rules = combinedRulesRoot.Children.Cast <GrammarASTWithOptions>().ToArray();
}
else
{
rules = new GrammarASTWithOptions[0];
}
foreach (GrammarASTWithOptions r in rules)
{
string ruleName = r.GetChild(0).Text;
if (Grammar.IsTokenName(ruleName))
{
lexerRulesRoot.AddChild((ITree)adaptor.DupTree(r));
rulesWeMoved.Add(r);
}
}
foreach (GrammarAST r in rulesWeMoved)
{
combinedRulesRoot.DeleteChild(r);
}
// Will track 'if' from IF : 'if' ; rules to avoid defining new token for 'if'
IList <System.Tuple <GrammarAST, GrammarAST> > litAliases =
Grammar.GetStringLiteralAliasesFromLexerRules(lexerAST);
ISet <string> stringLiterals = combinedGrammar.GetStringLiterals();
// add strings from combined grammar (and imported grammars) into lexer
// put them first as they are keywords; must resolve ambigs to these rules
// tool.log("grammar", "strings from parser: "+stringLiterals);
int insertIndex = 0;
foreach (string lit in stringLiterals)
{
// if lexer already has a rule for literal, continue
if (litAliases != null)
{
foreach (System.Tuple <GrammarAST, GrammarAST> pair in litAliases)
{
GrammarAST litAST = pair.Item2;
if (lit.Equals(litAST.Text))
{
goto continueNextLit;
}
}
}
// create for each literal: (RULE <uniquename> (BLOCK (ALT <lit>))
string rname = combinedGrammar.GetStringLiteralLexerRuleName(lit);
// can't use wizard; need special node types
GrammarAST litRule = new RuleAST(ANTLRParser.RULE);
BlockAST blk = new BlockAST(ANTLRParser.BLOCK);
AltAST alt = new AltAST(ANTLRParser.ALT);
TerminalAST slit = new TerminalAST(new CommonToken(ANTLRParser.STRING_LITERAL, lit));
alt.AddChild(slit);
blk.AddChild(alt);
CommonToken idToken = new CommonToken(ANTLRParser.TOKEN_REF, rname);
litRule.AddChild(new TerminalAST(idToken));
litRule.AddChild(blk);
lexerRulesRoot.InsertChild(insertIndex, litRule);
// lexerRulesRoot.getChildren().add(0, litRule);
lexerRulesRoot.FreshenParentAndChildIndexes(); // reset indexes and set litRule parent
// next literal will be added after the one just added
insertIndex++;
continueNextLit:
;
}
// TODO: take out after stable if slow
lexerAST.SanityCheckParentAndChildIndexes();
combinedAST.SanityCheckParentAndChildIndexes();
// tool.log("grammar", combinedAST.toTokenString());
combinedGrammar.tool.Log("grammar", "after extract implicit lexer =" + combinedAST.ToStringTree());
combinedGrammar.tool.Log("grammar", "lexer =" + lexerAST.ToStringTree());
if (lexerRulesRoot.ChildCount == 0)
{
return(null);
}
return(lexerAST);
}
/** Build lexer grammar from combined grammar that looks like:
*
* (COMBINED_GRAMMAR A
* (tokens { X (= Y 'y'))
* (OPTIONS (= x 'y'))
* (@ members {foo})
* (@ lexer header {package jj;})
* (RULES (RULE .+)))
*
* Move rules and actions to new tree, don't dup. Split AST apart.
* We'll have this Grammar share token symbols later; don't generate
* tokenVocab or tokens{} section. Copy over named actions.
*
* Side-effects: it removes children from GRAMMAR & RULES nodes
* in combined AST. Anything cut out is dup'd before
* adding to lexer to avoid "who's ur daddy" issues
*/
public virtual GrammarRootAST ExtractImplicitLexer(Grammar combinedGrammar)
{
GrammarRootAST combinedAST = combinedGrammar.ast;
//tool.log("grammar", "before="+combinedAST.toStringTree());
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(combinedAST.Token.InputStream);
GrammarAST[] elements = combinedAST.GetChildrenAsArray();
// MAKE A GRAMMAR ROOT and ID
string lexerName = combinedAST.GetChild(0).Text + "Lexer";
GrammarRootAST lexerAST =
new GrammarRootAST(new CommonToken(ANTLRParser.GRAMMAR, "LEXER_GRAMMAR"), combinedGrammar.ast.tokenStream);
lexerAST.grammarType = ANTLRParser.LEXER;
lexerAST.Token.InputStream = combinedAST.Token.InputStream;
lexerAST.AddChild((ITree)adaptor.Create(ANTLRParser.ID, lexerName));
// COPY OPTIONS
GrammarAST optionsRoot =
(GrammarAST)combinedAST.GetFirstChildWithType(ANTLRParser.OPTIONS);
if (optionsRoot != null && optionsRoot.ChildCount != 0)
{
GrammarAST lexerOptionsRoot = (GrammarAST)adaptor.DupNode(optionsRoot);
lexerAST.AddChild(lexerOptionsRoot);
GrammarAST[] options = optionsRoot.GetChildrenAsArray();
foreach (GrammarAST o in options)
{
string optionName = o.GetChild(0).Text;
if (Grammar.lexerOptions.Contains(optionName) &&
!Grammar.doNotCopyOptionsToLexer.Contains(optionName))
{
GrammarAST optionTree = (GrammarAST)adaptor.DupTree(o);
lexerOptionsRoot.AddChild(optionTree);
lexerAST.SetOption(optionName, (GrammarAST)optionTree.GetChild(1));
}
}
}
// COPY all named actions, but only move those with lexer:: scope
IList<GrammarAST> actionsWeMoved = new List<GrammarAST>();
foreach (GrammarAST e in elements)
{
if (e.Type == ANTLRParser.AT)
{
lexerAST.AddChild((ITree)adaptor.DupTree(e));
if (e.GetChild(0).Text.Equals("lexer"))
{
actionsWeMoved.Add(e);
}
}
}
foreach (GrammarAST r in actionsWeMoved)
{
combinedAST.DeleteChild(r);
}
GrammarAST combinedRulesRoot =
(GrammarAST)combinedAST.GetFirstChildWithType(ANTLRParser.RULES);
if (combinedRulesRoot == null)
return lexerAST;
// MOVE lexer rules
GrammarAST lexerRulesRoot = (GrammarAST)adaptor.Create(ANTLRParser.RULES, "RULES");
lexerAST.AddChild(lexerRulesRoot);
IList<GrammarAST> rulesWeMoved = new List<GrammarAST>();
GrammarASTWithOptions[] rules;
if (combinedRulesRoot.ChildCount > 0)
{
rules = combinedRulesRoot.Children.Cast<GrammarASTWithOptions>().ToArray();
}
else
{
rules = new GrammarASTWithOptions[0];
}
foreach (GrammarASTWithOptions r in rules)
{
string ruleName = r.GetChild(0).Text;
if (Grammar.IsTokenName(ruleName))
{
lexerRulesRoot.AddChild((ITree)adaptor.DupTree(r));
rulesWeMoved.Add(r);
}
}
foreach (GrammarAST r in rulesWeMoved)
{
combinedRulesRoot.DeleteChild(r);
}
// Will track 'if' from IF : 'if' ; rules to avoid defining new token for 'if'
IList<System.Tuple<GrammarAST, GrammarAST>> litAliases =
Grammar.GetStringLiteralAliasesFromLexerRules(lexerAST);
ISet<string> stringLiterals = combinedGrammar.GetStringLiterals();
// add strings from combined grammar (and imported grammars) into lexer
// put them first as they are keywords; must resolve ambigs to these rules
// tool.log("grammar", "strings from parser: "+stringLiterals);
int insertIndex = 0;
foreach (string lit in stringLiterals)
{
// if lexer already has a rule for literal, continue
if (litAliases != null)
{
foreach (System.Tuple<GrammarAST, GrammarAST> pair in litAliases)
{
GrammarAST litAST = pair.Item2;
if (lit.Equals(litAST.Text))
goto continueNextLit;
}
}
// create for each literal: (RULE <uniquename> (BLOCK (ALT <lit>))
string rname = combinedGrammar.GetStringLiteralLexerRuleName(lit);
// can't use wizard; need special node types
GrammarAST litRule = new RuleAST(ANTLRParser.RULE);
BlockAST blk = new BlockAST(ANTLRParser.BLOCK);
AltAST alt = new AltAST(ANTLRParser.ALT);
TerminalAST slit = new TerminalAST(new CommonToken(ANTLRParser.STRING_LITERAL, lit));
alt.AddChild(slit);
blk.AddChild(alt);
CommonToken idToken = new CommonToken(ANTLRParser.TOKEN_REF, rname);
litRule.AddChild(new TerminalAST(idToken));
litRule.AddChild(blk);
lexerRulesRoot.InsertChild(insertIndex, litRule);
// lexerRulesRoot.getChildren().add(0, litRule);
lexerRulesRoot.FreshenParentAndChildIndexes(); // reset indexes and set litRule parent
// next literal will be added after the one just added
insertIndex++;
continueNextLit:
;
}
// TODO: take out after stable if slow
lexerAST.SanityCheckParentAndChildIndexes();
combinedAST.SanityCheckParentAndChildIndexes();
// tool.log("grammar", combinedAST.toTokenString());
combinedGrammar.tool.Log("grammar", "after extract implicit lexer =" + combinedAST.ToStringTree());
combinedGrammar.tool.Log("grammar", "lexer =" + lexerAST.ToStringTree());
if (lexerRulesRoot.ChildCount == 0)
return null;
return lexerAST;
}
public override void ConsumeToken(IToken token)
{
lastTokenConsumed = (CommonToken)token;
}
public V(int ttype, int x)
{
this.x = x;
Token = new CommonToken(ttype);
}
public override void ConsumeHiddenToken(IToken token)
{
lastTokenConsumed = (CommonToken)token;
}
public virtual void Initialize(int i, string s)
{
token = new CommonToken(i, s);
}
