public static void wipeCharPositionInfoAndWhitespaceTokens(CodeBuffTokenStream tokens) { tokens.Fill(); CommonToken dummy = new CommonToken(TokenConstants.InvalidType, ""); dummy.Channel = TokenConstants.HiddenChannel; Token firstRealToken = tokens.getNextRealToken(-1); for (int i = 0; i < tokens.Size; i++) { if (i == firstRealToken.TokenIndex) { continue; // don't wack first token } CommonToken t = (CommonToken)tokens.Get(i); Regex rex = new Regex("^\\s+$"); if (rex.IsMatch(t.Text)) { tokens.GetTokens()[i] = dummy; // wack whitespace token so we can't use it during prediction } else { t.Line = 0; t.Column = -1; } } }
/// <summary> /// Return map for the various tokens related to this list re list membership </summary> public static IDictionary <Token, org.antlr.codebuff.misc.Pair <bool, int> > getInfoAboutListTokens <T1>(ParserRuleContext ctx, CodeBuffTokenStream tokens, IDictionary <Token, TerminalNode> tokenToNodeMap, IList <T1> siblings, bool isOversizeList) where T1 : Antlr4.Runtime.ParserRuleContext { IDictionary <Token, org.antlr.codebuff.misc.Pair <bool, int> > tokenToListInfo = new Dictionary <Token, org.antlr.codebuff.misc.Pair <bool, int> >(); ParserRuleContext first = siblings[0] as ParserRuleContext; ParserRuleContext last = siblings[siblings.Count - 1] as ParserRuleContext; Token prefixToken = tokens.getPreviousRealToken(first.Start.TokenIndex); // e.g., '(' in an arg list or ':' in grammar def Token suffixToken = tokens.getNextRealToken(last.Stop.TokenIndex); // e.g., LT(1) is last token of list; LT(2) is ')' in an arg list of ';' in grammar def if (prefixToken != null && suffixToken != null) { TerminalNode prefixNode = tokenToNodeMap[prefixToken]; TerminalNode suffixNode = tokenToNodeMap[suffixToken]; bool hasSurroundingTokens = prefixNode != null && prefixNode.Parent == suffixNode.Parent; if (hasSurroundingTokens) { tokenToListInfo[prefixToken] = new org.antlr.codebuff.misc.Pair <bool, int>(isOversizeList, Trainer.LIST_PREFIX); tokenToListInfo[suffixToken] = new org.antlr.codebuff.misc.Pair <bool, int>(isOversizeList, Trainer.LIST_SUFFIX); } IList <Tree> separators = getSeparators(ctx, siblings); Tree firstSep = separators[0]; tokenToListInfo[(Token)firstSep.Payload] = new org.antlr.codebuff.misc.Pair <bool, int>(isOversizeList, Trainer.LIST_FIRST_SEPARATOR); foreach (Tree s in separators.Where((e, i) => i > 0 && i < separators.Count)) { tokenToListInfo[(Token)s.Payload] = new org.antlr.codebuff.misc.Pair <bool, int>(isOversizeList, Trainer.LIST_SEPARATOR); } // handle sibling members tokenToListInfo[first.Start] = new org.antlr.codebuff.misc.Pair <bool, int>(isOversizeList, Trainer.LIST_FIRST_ELEMENT); foreach (T1 ss in siblings.Where((e, i) => i > 0 && i < siblings.Count)) { var s = ss as ParserRuleContext; tokenToListInfo[s.Start] = new org.antlr.codebuff.misc.Pair <bool, int>(isOversizeList, Trainer.LIST_MEMBER); } } return(tokenToListInfo); }