public virtual RuleAST ParseArtificialRule(Grammar g, string ruleText)
{
ANTLRLexer lexer = new ANTLRLexer(new ANTLRStringStream(ruleText));
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(lexer.CharStream);
CommonTokenStream tokens = new CommonTokenStream(lexer);
lexer.tokens = tokens;
ToolANTLRParser p = new ToolANTLRParser(tokens, tool);
p.TreeAdaptor = adaptor;
IToken ruleStart = null;
try
{
Antlr.Runtime.AstParserRuleReturnScope <GrammarAST, IToken> r = p.rule();
RuleAST tree = (RuleAST)r.Tree;
ruleStart = r.Start;
GrammarTransformPipeline.SetGrammarPtr(g, tree);
GrammarTransformPipeline.AugmentTokensWithOriginalPosition(g, tree);
return(tree);
}
catch (Exception e)
{
tool.errMgr.ToolError(ErrorType.INTERNAL_ERROR,
e,
ruleStart,
"error parsing rule created during left-recursion detection: " + ruleText);
}
return(null);
}
private List <LabelInfo> FindLabelsInScope(SnapshotPoint triggerPoint)
{
List <LabelInfo> labels = new List <LabelInfo>();
/* use the experimental model to locate and process the expression */
Stopwatch stopwatch = Stopwatch.StartNew();
// lex the entire document
var currentSnapshot = triggerPoint.Snapshot;
var input = new SnapshotCharStream(currentSnapshot, new Span(0, currentSnapshot.Length));
var lexer = new ANTLRLexer(input);
var tokens = new CommonTokenStream(lexer);
tokens.Fill();
// locate the last token before the trigger point
while (true)
{
IToken nextToken = tokens.Lt(1);
if (nextToken.Type == CharStreamConstants.EndOfFile)
{
break;
}
if (nextToken.StartIndex > triggerPoint.Position)
{
break;
}
tokens.Consume();
}
bool inAction = false;
IToken triggerToken = tokens.LT(-1);
switch (triggerToken.Type)
{
case ANTLRLexer.RULE_REF:
case ANTLRLexer.TOKEN_REF:
case ANTLRLexer.DOLLAR:
break;
case ANTLRLexer.ACTION:
case ANTLRLexer.FORCED_ACTION:
case ANTLRLexer.SEMPRED:
case ANTLRLexer.ARG_ACTION:
inAction = true;
break;
default:
return(labels);
}
NetworkInterpreter interpreter = CreateNetworkInterpreter(tokens);
while (interpreter.TryStepBackward())
{
if (interpreter.Contexts.Count == 0 || interpreter.Contexts.Count > 4000)
{
break;
}
if (interpreter.Contexts.All(context => context.BoundedStart))
{
break;
}
}
if (interpreter.Failed)
{
interpreter.Contexts.Clear();
}
interpreter.CombineBoundedStartContexts();
HashSet <IToken> labelTokens = new HashSet <IToken>(TokenIndexEqualityComparer.Default);
foreach (var context in interpreter.Contexts)
{
var tokenTransitions = context.Transitions.Where(i => i.TokenIndex != null).ToList();
for (int i = 1; i < tokenTransitions.Count - 1; i++)
{
if (tokenTransitions[i].Symbol != ANTLRLexer.TOKEN_REF && tokenTransitions[i].Symbol != ANTLRLexer.RULE_REF)
{
continue;
}
// we add explicit labels, plus implicit labels if we're in an action
if (tokenTransitions[i + 1].Symbol == ANTLRLexer.ASSIGN || tokenTransitions[i + 1].Symbol == ANTLRLexer.PLUS_ASSIGN)
{
RuleBinding rule = interpreter.Network.StateRules[tokenTransitions[i + 1].Transition.SourceState.Id];
if (rule.Name == AntlrAtnBuilder.RuleNames.TreeRoot || rule.Name == AntlrAtnBuilder.RuleNames.ElementNoOptionSpec)
{
labelTokens.Add(tokenTransitions[i].Token);
}
}
else if (inAction && tokenTransitions[i - 1].Symbol != ANTLRLexer.ASSIGN && tokenTransitions[i - 1].Symbol != ANTLRLexer.PLUS_ASSIGN)
{
RuleBinding rule = interpreter.Network.StateRules[tokenTransitions[i].Transition.SourceState.Id];
if (rule.Name == AntlrAtnBuilder.RuleNames.Terminal || rule.Name == AntlrAtnBuilder.RuleNames.NotTerminal || rule.Name == AntlrAtnBuilder.RuleNames.RuleRef)
{
labelTokens.Add(tokenTransitions[i].Token);
}
}
}
}
foreach (var token in labelTokens)
{
labels.Add(new LabelInfo(token.Text, "(label) " + token.Text, new SnapshotSpan(triggerPoint.Snapshot, Span.FromBounds(token.StartIndex, token.StopIndex + 1)), StandardGlyphGroup.GlyphGroupField, Enumerable.Empty <LabelInfo>()));
}
/* add scopes */
if (inAction)
{
/* add global scopes */
IList <IToken> tokensList = tokens.GetTokens();
for (int i = 0; i < tokensList.Count - 1; i++)
{
var token = tokensList[i];
/* all global scopes appear before the first rule. before the first rule, the only place a ':' can appear is
* in the form '::' for things like @lexer::namespace{}
*/
if (token.Type == ANTLRLexer.COLON && tokensList[i + 1].Type == ANTLRLexer.COLON)
{
break;
}
if (token.Type == ANTLRLexer.SCOPE)
{
var nextToken = tokensList.Skip(i + 1).FirstOrDefault(t => t.Channel == TokenChannels.Default);
if (nextToken != null && (nextToken.Type == ANTLRLexer.RULE_REF || nextToken.Type == ANTLRLexer.TOKEN_REF))
{
// TODO: parse scope members
IToken actionToken = tokensList.Skip(nextToken.TokenIndex + 1).FirstOrDefault(t => t.Channel == TokenChannels.Default);
IEnumerable <LabelInfo> members = Enumerable.Empty <LabelInfo>();
if (actionToken != null && actionToken.Type == ANTLRLexer.ACTION)
{
IEnumerable <IToken> scopeMembers = ExtractScopeAttributes(nextToken);
members = scopeMembers.Select(member =>
{
string name = member.Text;
SnapshotSpan definition = new SnapshotSpan(triggerPoint.Snapshot, Span.FromBounds(member.StartIndex, member.StopIndex + 1));
StandardGlyphGroup glyph = StandardGlyphGroup.GlyphGroupField;
IEnumerable <LabelInfo> nestedMembers = Enumerable.Empty <LabelInfo>();
return(new LabelInfo(name, string.Empty, definition, glyph, nestedMembers));
});
}
labels.Add(new LabelInfo(nextToken.Text, "(global scope) " + nextToken.Text, new SnapshotSpan(triggerPoint.Snapshot, Span.FromBounds(nextToken.StartIndex, nextToken.StopIndex + 1)), StandardGlyphGroup.GlyphGroupNamespace, members));
}
}
}
/* add rule scopes */
// todo
}
/* add arguments and return values */
if (inAction)
{
HashSet <IToken> argumentTokens = new HashSet <IToken>(TokenIndexEqualityComparer.Default);
foreach (var context in interpreter.Contexts)
{
var tokenTransitions = context.Transitions.Where(i => i.TokenIndex != null).ToList();
for (int i = 1; i < tokenTransitions.Count; i++)
{
if (tokenTransitions[i].Symbol == ANTLRLexer.RETURNS || tokenTransitions[i].Symbol == ANTLRLexer.COLON)
{
break;
}
if (tokenTransitions[i].Symbol == ANTLRLexer.ARG_ACTION)
{
argumentTokens.Add(tokenTransitions[i].Token);
}
}
}
foreach (var token in argumentTokens)
{
IEnumerable <IToken> arguments = ExtractArguments(token);
foreach (var argument in arguments)
{
labels.Add(new LabelInfo(argument.Text, "(parameter) " + argument.Text, new SnapshotSpan(triggerPoint.Snapshot, Span.FromBounds(argument.StartIndex, argument.StopIndex + 1)), StandardGlyphGroup.GlyphGroupVariable, Enumerable.Empty <LabelInfo>()));
}
}
}
/* add return values */
if (inAction)
{
HashSet <IToken> returnTokens = new HashSet <IToken>(TokenIndexEqualityComparer.Default);
foreach (var context in interpreter.Contexts)
{
var tokenTransitions = context.Transitions.Where(i => i.TokenIndex != null).ToList();
for (int i = 1; i < tokenTransitions.Count - 1; i++)
{
if (tokenTransitions[i].Symbol == ANTLRLexer.COLON)
{
break;
}
if (tokenTransitions[i].Symbol == ANTLRLexer.RETURNS)
{
if (tokenTransitions[i + 1].Symbol == ANTLRLexer.ARG_ACTION)
{
returnTokens.Add(tokenTransitions[i + 1].Token);
}
break;
}
}
}
foreach (var token in returnTokens)
{
IEnumerable <IToken> returnValues = ExtractArguments(token);
foreach (var returnValue in returnValues)
{
labels.Add(new LabelInfo(returnValue.Text, "(return value) " + returnValue.Text, new SnapshotSpan(triggerPoint.Snapshot, Span.FromBounds(returnValue.StartIndex, returnValue.StopIndex + 1)), StandardGlyphGroup.GlyphGroupVariable, Enumerable.Empty <LabelInfo>()));
}
}
}
/* add intrinsic labels ($start, $type, $text, $enclosingRuleName) */
IToken ruleNameToken = null;
HashSet <IToken> enclosingRuleNameTokens = new HashSet <IToken>(TokenIndexEqualityComparer.Default);
foreach (var context in interpreter.Contexts)
{
var tokenTransitions = context.Transitions.Where(i => i.Symbol == ANTLRLexer.RULE_REF || i.Symbol == ANTLRLexer.TOKEN_REF).ToList();
if (!tokenTransitions.Any())
{
continue;
}
ruleNameToken = tokenTransitions.First().Token;
if (ruleNameToken != null)
{
enclosingRuleNameTokens.Add(ruleNameToken);
}
}
foreach (var token in enclosingRuleNameTokens)
{
// TODO: add members
labels.Add(new LabelInfo(token.Text, "(enclosing rule) " + token.Text, new SnapshotSpan(triggerPoint.Snapshot, Span.FromBounds(token.StartIndex, token.StopIndex + 1)), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
}
GrammarType grammarType = GrammarType.None;
int mark = tokens.Mark();
try
{
tokens.Seek(0);
bool hasGrammarType = false;
while (!hasGrammarType)
{
int la1 = tokens.LA(1);
switch (la1)
{
case ANTLRLexer.GRAMMAR:
IToken previous = tokens.LT(-1);
if (previous == null)
{
grammarType = GrammarType.Combined;
}
else if (previous.Type == ANTLRLexer.LEXER)
{
grammarType = GrammarType.Lexer;
}
else if (previous.Type == ANTLRLexer.PARSER)
{
grammarType = GrammarType.Parser;
}
else if (previous.Type == ANTLRLexer.TREE)
{
grammarType = GrammarType.TreeParser;
}
else
{
grammarType = GrammarType.None;
}
hasGrammarType = true;
break;
case CharStreamConstants.EndOfFile:
hasGrammarType = true;
break;
default:
break;
}
tokens.Consume();
}
}
finally
{
tokens.Rewind(mark);
}
if (inAction)
{
switch (grammarType)
{
case GrammarType.Combined:
if (ruleNameToken == null)
{
goto default;
}
if (ruleNameToken.Type == ANTLRLexer.RULE_REF)
{
goto case GrammarType.Parser;
}
else
{
goto case GrammarType.Lexer;
}
case GrammarType.Lexer:
labels.Add(new LabelInfo("text", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("type", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("line", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("index", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("pos", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("channel", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("start", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("stop", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("int", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
break;
case GrammarType.Parser:
labels.Add(new LabelInfo("text", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("start", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("stop", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("tree", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("st", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
break;
case GrammarType.TreeParser:
labels.Add(new LabelInfo("text", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("start", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("tree", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("st", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
break;
default:
// if we're unsure about the grammar type, include all the possible options to make sure we're covered
labels.Add(new LabelInfo("text", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("type", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("line", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("index", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("pos", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("channel", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("start", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("stop", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("int", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("tree", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
labels.Add(new LabelInfo("st", string.Empty, new SnapshotSpan(), StandardGlyphGroup.GlyphGroupIntrinsic, Enumerable.Empty <LabelInfo>()));
break;
}
}
return(labels);
}
public virtual RuleAST ParseArtificialRule(Grammar g, string ruleText)
{
ANTLRLexer lexer = new ANTLRLexer(new ANTLRStringStream(ruleText));
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(lexer.CharStream);
CommonTokenStream tokens = new CommonTokenStream(lexer);
lexer.tokens = tokens;
ToolANTLRParser p = new ToolANTLRParser(tokens, tool);
p.TreeAdaptor = adaptor;
IToken ruleStart = null;
try
{
Antlr.Runtime.AstParserRuleReturnScope<GrammarAST, IToken> r = p.rule();
RuleAST tree = (RuleAST)r.Tree;
ruleStart = r.Start;
GrammarTransformPipeline.SetGrammarPtr(g, tree);
GrammarTransformPipeline.AugmentTokensWithOriginalPosition(g, tree);
return tree;
}
catch (Exception e)
{
tool.errMgr.ToolError(ErrorType.INTERNAL_ERROR,
e,
ruleStart,
"error parsing rule created during left-recursion detection: " + ruleText);
}
return null;
}
