public virtual IList <IToken> Tokenize(string pattern)
{
// split pattern into chunks: sea (raw input) and islands (<ID>, <expr>)
IList <Chunk> chunks = Split(pattern);
// create token stream from text and tags
IList <IToken> tokens = new List <IToken>();
foreach (Chunk chunk in chunks)
{
if (chunk is TagChunk)
{
TagChunk tagChunk = (TagChunk)chunk;
// add special rule token or conjure up new token from name
if (System.Char.IsUpper(tagChunk.Tag[0]))
{
int ttype = parser.GetTokenType(tagChunk.Tag);
if (ttype == TokenConstants.InvalidType)
{
throw new ArgumentException("Unknown token " + tagChunk.Tag + " in pattern: " + pattern);
}
TokenTagToken t = new TokenTagToken(tagChunk.Tag, ttype, tagChunk.Label);
tokens.Add(t);
}
else
{
if (System.Char.IsLower(tagChunk.Tag[0]))
{
int ruleIndex = parser.GetRuleIndex(tagChunk.Tag);
if (ruleIndex == -1)
{
throw new ArgumentException("Unknown rule " + tagChunk.Tag + " in pattern: " + pattern);
}
int ruleImaginaryTokenType = parser.GetATNWithBypassAlts().ruleToTokenType[ruleIndex];
tokens.Add(new RuleTagToken(tagChunk.Tag, ruleImaginaryTokenType, tagChunk.Label));
}
else
{
throw new ArgumentException("invalid tag: " + tagChunk.Tag + " in pattern: " + pattern);
}
}
}
else
{
TextChunk textChunk = (TextChunk)chunk;
AntlrInputStream @in = new AntlrInputStream(textChunk.Text);
lexer.SetInputStream(@in);
IToken t = lexer.NextToken();
while (t.Type != TokenConstants.EOF)
{
tokens.Add(t);
t = lexer.NextToken();
}
}
}
// System.out.println("tokens="+tokens);
return(tokens);
}
public virtual IList<IToken> Tokenize(string pattern)
{
// split pattern into chunks: sea (raw input) and islands (<ID>, <expr>)
IList<Chunk> chunks = Split(pattern);
// create token stream from text and tags
IList<IToken> tokens = new List<IToken>();
foreach (Chunk chunk in chunks)
{
if (chunk is TagChunk)
{
TagChunk tagChunk = (TagChunk)chunk;
// add special rule token or conjure up new token from name
if (System.Char.IsUpper(tagChunk.Tag[0]))
{
int ttype = parser.GetTokenType(tagChunk.Tag);
if (ttype == TokenConstants.InvalidType)
{
throw new ArgumentException("Unknown token " + tagChunk.Tag + " in pattern: " + pattern);
}
TokenTagToken t = new TokenTagToken(tagChunk.Tag, ttype, tagChunk.Label);
tokens.Add(t);
}
else
{
if (System.Char.IsLower(tagChunk.Tag[0]))
{
int ruleIndex = parser.GetRuleIndex(tagChunk.Tag);
if (ruleIndex == -1)
{
throw new ArgumentException("Unknown rule " + tagChunk.Tag + " in pattern: " + pattern);
}
int ruleImaginaryTokenType = parser.GetATNWithBypassAlts().ruleToTokenType[ruleIndex];
tokens.Add(new RuleTagToken(tagChunk.Tag, ruleImaginaryTokenType, tagChunk.Label));
}
else
{
throw new ArgumentException("invalid tag: " + tagChunk.Tag + " in pattern: " + pattern);
}
}
}
else
{
TextChunk textChunk = (TextChunk)chunk;
AntlrInputStream @in = new AntlrInputStream(textChunk.Text);
lexer.SetInputStream(@in);
IToken t = lexer.NextToken();
while (t.Type != TokenConstants.Eof)
{
tokens.Add(t);
t = lexer.NextToken();
}
}
}
// System.out.println("tokens="+tokens);
return tokens;
}
protected internal virtual IParseTree MatchImpl(IParseTree tree, IParseTree patternTree, MultiMap <string, IParseTree> labels)
{
if (tree == null)
{
throw new ArgumentException("tree cannot be null");
}
if (patternTree == null)
{
throw new ArgumentException("patternTree cannot be null");
}
// x and <ID>, x and y, or x and x; or could be mismatched types
if (tree is ITerminalNode && patternTree is ITerminalNode)
{
ITerminalNode t1 = (ITerminalNode)tree;
ITerminalNode t2 = (ITerminalNode)patternTree;
IParseTree mismatchedNode = null;
// both are tokens and they have same type
if (t1.Symbol.Type == t2.Symbol.Type)
{
if (t2.Symbol is TokenTagToken)
{
// x and <ID>
TokenTagToken tokenTagToken = (TokenTagToken)t2.Symbol;
// track label->list-of-nodes for both token name and label (if any)
labels.Map(tokenTagToken.TokenName, tree);
if (tokenTagToken.Label != null)
{
labels.Map(tokenTagToken.Label, tree);
}
}
else
{
if (t1.GetText().Equals(t2.GetText(), StringComparison.Ordinal))
{
}
else
{
// x and x
// x and y
if (mismatchedNode == null)
{
mismatchedNode = t1;
}
}
}
}
else
{
if (mismatchedNode == null)
{
mismatchedNode = t1;
}
}
return(mismatchedNode);
}
if (tree is ParserRuleContext && patternTree is ParserRuleContext)
{
ParserRuleContext r1 = (ParserRuleContext)tree;
ParserRuleContext r2 = (ParserRuleContext)patternTree;
IParseTree mismatchedNode = null;
// (expr ...) and <expr>
RuleTagToken ruleTagToken = GetRuleTagToken(r2);
if (ruleTagToken != null)
{
if (r1.RuleIndex == r2.RuleIndex)
{
// track label->list-of-nodes for both rule name and label (if any)
labels.Map(ruleTagToken.RuleName, tree);
if (ruleTagToken.Label != null)
{
labels.Map(ruleTagToken.Label, tree);
}
}
else
{
if (mismatchedNode == null)
{
mismatchedNode = r1;
}
}
return(mismatchedNode);
}
// (expr ...) and (expr ...)
if (r1.ChildCount != r2.ChildCount)
{
if (mismatchedNode == null)
{
mismatchedNode = r1;
}
return(mismatchedNode);
}
int n = r1.ChildCount;
for (int i = 0; i < n; i++)
{
IParseTree childMatch = MatchImpl(r1.GetChild(i), patternTree.GetChild(i), labels);
if (childMatch != null)
{
return(childMatch);
}
}
return(mismatchedNode);
}
// if nodes aren't both tokens or both rule nodes, can't match
return(tree);
}