public virtual IList <IToken> Tokenize(string pattern) { // split pattern into chunks: sea (raw input) and islands (<ID>, <expr>) IList <Chunk> chunks = Split(pattern); // create token stream from text and tags IList <IToken> tokens = new List <IToken>(); foreach (Chunk chunk in chunks) { if (chunk is TagChunk) { TagChunk tagChunk = (TagChunk)chunk; // add special rule token or conjure up new token from name if (System.Char.IsUpper(tagChunk.Tag[0])) { int ttype = parser.GetTokenType(tagChunk.Tag); if (ttype == TokenConstants.InvalidType) { throw new ArgumentException("Unknown token " + tagChunk.Tag + " in pattern: " + pattern); } TokenTagToken t = new TokenTagToken(tagChunk.Tag, ttype, tagChunk.Label); tokens.Add(t); } else { if (System.Char.IsLower(tagChunk.Tag[0])) { int ruleIndex = parser.GetRuleIndex(tagChunk.Tag); if (ruleIndex == -1) { throw new ArgumentException("Unknown rule " + tagChunk.Tag + " in pattern: " + pattern); } int ruleImaginaryTokenType = parser.GetATNWithBypassAlts().ruleToTokenType[ruleIndex]; tokens.Add(new RuleTagToken(tagChunk.Tag, ruleImaginaryTokenType, tagChunk.Label)); } else { throw new ArgumentException("invalid tag: " + tagChunk.Tag + " in pattern: " + pattern); } } } else { TextChunk textChunk = (TextChunk)chunk; AntlrInputStream @in = new AntlrInputStream(textChunk.Text); lexer.SetInputStream(@in); IToken t = lexer.NextToken(); while (t.Type != TokenConstants.EOF) { tokens.Add(t); t = lexer.NextToken(); } } } // System.out.println("tokens="+tokens); return(tokens); }
protected internal virtual IParseTree MatchImpl(IParseTree tree, IParseTree patternTree, MultiMap <string, IParseTree> labels) { if (tree == null) { throw new ArgumentException("tree cannot be null"); } if (patternTree == null) { throw new ArgumentException("patternTree cannot be null"); } // x and <ID>, x and y, or x and x; or could be mismatched types if (tree is ITerminalNode && patternTree is ITerminalNode) { ITerminalNode t1 = (ITerminalNode)tree; ITerminalNode t2 = (ITerminalNode)patternTree; IParseTree mismatchedNode = null; // both are tokens and they have same type if (t1.Symbol.Type == t2.Symbol.Type) { if (t2.Symbol is TokenTagToken) { // x and <ID> TokenTagToken tokenTagToken = (TokenTagToken)t2.Symbol; // track label->list-of-nodes for both token name and label (if any) labels.Map(tokenTagToken.TokenName, tree); if (tokenTagToken.Label != null) { labels.Map(tokenTagToken.Label, tree); } } else { if (t1.GetText().Equals(t2.GetText(), StringComparison.Ordinal)) { } else { // x and x // x and y if (mismatchedNode == null) { mismatchedNode = t1; } } } } else { if (mismatchedNode == null) { mismatchedNode = t1; } } return(mismatchedNode); } if (tree is ParserRuleContext && patternTree is ParserRuleContext) { ParserRuleContext r1 = (ParserRuleContext)tree; ParserRuleContext r2 = (ParserRuleContext)patternTree; IParseTree mismatchedNode = null; // (expr ...) and <expr> RuleTagToken ruleTagToken = GetRuleTagToken(r2); if (ruleTagToken != null) { if (r1.RuleIndex == r2.RuleIndex) { // track label->list-of-nodes for both rule name and label (if any) labels.Map(ruleTagToken.RuleName, tree); if (ruleTagToken.Label != null) { labels.Map(ruleTagToken.Label, tree); } } else { if (mismatchedNode == null) { mismatchedNode = r1; } } return(mismatchedNode); } // (expr ...) and (expr ...) if (r1.ChildCount != r2.ChildCount) { if (mismatchedNode == null) { mismatchedNode = r1; } return(mismatchedNode); } int n = r1.ChildCount; for (int i = 0; i < n; i++) { IParseTree childMatch = MatchImpl(r1.GetChild(i), patternTree.GetChild(i), labels); if (childMatch != null) { return(childMatch); } } return(mismatchedNode); } // if nodes aren't both tokens or both rule nodes, can't match return(tree); }