public static void wipeCharPositionInfoAndWhitespaceTokens(CodeBuffTokenStream tokens)
{
tokens.Fill();
CommonToken dummy = new CommonToken(TokenConstants.InvalidType, "");
dummy.Channel = TokenConstants.HiddenChannel;
Token firstRealToken = tokens.getNextRealToken(-1);
for (int i = 0; i < tokens.Size; i++)
{
if (i == firstRealToken.TokenIndex)
{
continue; // don't wack first token
}
CommonToken t = (CommonToken)tokens.Get(i);
Regex rex = new Regex("^\\s+$");
if (rex.IsMatch(t.Text))
{
tokens.GetTokens()[i] = dummy;
// wack whitespace token so we can't use it during prediction
}
else
{
t.Line = 0;
t.Column = -1;
}
}
}
/// <summary>
/// Return map for the various tokens related to this list re list membership </summary>
public static IDictionary <Token, org.antlr.codebuff.misc.Pair <bool, int> > getInfoAboutListTokens <T1>(ParserRuleContext ctx, CodeBuffTokenStream tokens, IDictionary <Token, TerminalNode> tokenToNodeMap, IList <T1> siblings, bool isOversizeList) where T1 : Antlr4.Runtime.ParserRuleContext
{
IDictionary <Token, org.antlr.codebuff.misc.Pair <bool, int> > tokenToListInfo = new Dictionary <Token, org.antlr.codebuff.misc.Pair <bool, int> >();
ParserRuleContext first = siblings[0] as ParserRuleContext;
ParserRuleContext last = siblings[siblings.Count - 1] as ParserRuleContext;
Token prefixToken = tokens.getPreviousRealToken(first.Start.TokenIndex); // e.g., '(' in an arg list or ':' in grammar def
Token suffixToken = tokens.getNextRealToken(last.Stop.TokenIndex); // e.g., LT(1) is last token of list; LT(2) is ')' in an arg list of ';' in grammar def
if (prefixToken != null && suffixToken != null)
{
TerminalNode prefixNode = tokenToNodeMap[prefixToken];
TerminalNode suffixNode = tokenToNodeMap[suffixToken];
bool hasSurroundingTokens = prefixNode != null && prefixNode.Parent == suffixNode.Parent;
if (hasSurroundingTokens)
{
tokenToListInfo[prefixToken] = new org.antlr.codebuff.misc.Pair <bool, int>(isOversizeList, Trainer.LIST_PREFIX);
tokenToListInfo[suffixToken] = new org.antlr.codebuff.misc.Pair <bool, int>(isOversizeList, Trainer.LIST_SUFFIX);
}
IList <Tree> separators = getSeparators(ctx, siblings);
Tree firstSep = separators[0];
tokenToListInfo[(Token)firstSep.Payload] = new org.antlr.codebuff.misc.Pair <bool, int>(isOversizeList, Trainer.LIST_FIRST_SEPARATOR);
foreach (Tree s in separators.Where((e, i) => i > 0 && i < separators.Count))
{
tokenToListInfo[(Token)s.Payload] = new org.antlr.codebuff.misc.Pair <bool, int>(isOversizeList, Trainer.LIST_SEPARATOR);
}
// handle sibling members
tokenToListInfo[first.Start] = new org.antlr.codebuff.misc.Pair <bool, int>(isOversizeList, Trainer.LIST_FIRST_ELEMENT);
foreach (T1 ss in siblings.Where((e, i) => i > 0 && i < siblings.Count))
{
var s = ss as ParserRuleContext;
tokenToListInfo[s.Start] = new org.antlr.codebuff.misc.Pair <bool, int>(isOversizeList, Trainer.LIST_MEMBER);
}
}
return(tokenToListInfo);
}
