Inheritance: Antlr4.Runtime.CommonToken
Ejemplo n.º 1
0
        public virtual IList <IToken> Tokenize(string pattern)
        {
            // split pattern into chunks: sea (raw input) and islands (<ID>, <expr>)
            IList <Chunk> chunks = Split(pattern);
            // create token stream from text and tags
            IList <IToken> tokens = new List <IToken>();

            foreach (Chunk chunk in chunks)
            {
                if (chunk is TagChunk)
                {
                    TagChunk tagChunk = (TagChunk)chunk;
                    // add special rule token or conjure up new token from name
                    if (System.Char.IsUpper(tagChunk.Tag[0]))
                    {
                        int ttype = parser.GetTokenType(tagChunk.Tag);
                        if (ttype == TokenConstants.InvalidType)
                        {
                            throw new ArgumentException("Unknown token " + tagChunk.Tag + " in pattern: " + pattern);
                        }
                        TokenTagToken t = new TokenTagToken(tagChunk.Tag, ttype, tagChunk.Label);
                        tokens.Add(t);
                    }
                    else
                    {
                        if (System.Char.IsLower(tagChunk.Tag[0]))
                        {
                            int ruleIndex = parser.GetRuleIndex(tagChunk.Tag);
                            if (ruleIndex == -1)
                            {
                                throw new ArgumentException("Unknown rule " + tagChunk.Tag + " in pattern: " + pattern);
                            }
                            int ruleImaginaryTokenType = parser.GetATNWithBypassAlts().ruleToTokenType[ruleIndex];
                            tokens.Add(new RuleTagToken(tagChunk.Tag, ruleImaginaryTokenType, tagChunk.Label));
                        }
                        else
                        {
                            throw new ArgumentException("invalid tag: " + tagChunk.Tag + " in pattern: " + pattern);
                        }
                    }
                }
                else
                {
                    TextChunk        textChunk = (TextChunk)chunk;
                    AntlrInputStream @in       = new AntlrInputStream(textChunk.Text);
                    lexer.SetInputStream(@in);
                    IToken t = lexer.NextToken();
                    while (t.Type != TokenConstants.EOF)
                    {
                        tokens.Add(t);
                        t = lexer.NextToken();
                    }
                }
            }
            //		System.out.println("tokens="+tokens);
            return(tokens);
        }
 public virtual IList<IToken> Tokenize(string pattern)
 {
     // split pattern into chunks: sea (raw input) and islands (<ID>, <expr>)
     IList<Chunk> chunks = Split(pattern);
     // create token stream from text and tags
     IList<IToken> tokens = new List<IToken>();
     foreach (Chunk chunk in chunks)
     {
         if (chunk is TagChunk)
         {
             TagChunk tagChunk = (TagChunk)chunk;
             // add special rule token or conjure up new token from name
             if (System.Char.IsUpper(tagChunk.Tag[0]))
             {
                 int ttype = parser.GetTokenType(tagChunk.Tag);
                 if (ttype == TokenConstants.InvalidType)
                 {
                     throw new ArgumentException("Unknown token " + tagChunk.Tag + " in pattern: " + pattern);
                 }
                 TokenTagToken t = new TokenTagToken(tagChunk.Tag, ttype, tagChunk.Label);
                 tokens.Add(t);
             }
             else
             {
                 if (System.Char.IsLower(tagChunk.Tag[0]))
                 {
                     int ruleIndex = parser.GetRuleIndex(tagChunk.Tag);
                     if (ruleIndex == -1)
                     {
                         throw new ArgumentException("Unknown rule " + tagChunk.Tag + " in pattern: " + pattern);
                     }
                     int ruleImaginaryTokenType = parser.GetATNWithBypassAlts().ruleToTokenType[ruleIndex];
                     tokens.Add(new RuleTagToken(tagChunk.Tag, ruleImaginaryTokenType, tagChunk.Label));
                 }
                 else
                 {
                     throw new ArgumentException("invalid tag: " + tagChunk.Tag + " in pattern: " + pattern);
                 }
             }
         }
         else
         {
             TextChunk textChunk = (TextChunk)chunk;
             AntlrInputStream @in = new AntlrInputStream(textChunk.Text);
             lexer.SetInputStream(@in);
             IToken t = lexer.NextToken();
             while (t.Type != TokenConstants.Eof)
             {
                 tokens.Add(t);
                 t = lexer.NextToken();
             }
         }
     }
     //		System.out.println("tokens="+tokens);
     return tokens;
 }
Ejemplo n.º 3
0
        protected internal virtual IParseTree MatchImpl(IParseTree tree, IParseTree patternTree, MultiMap <string, IParseTree> labels)
        {
            if (tree == null)
            {
                throw new ArgumentException("tree cannot be null");
            }
            if (patternTree == null)
            {
                throw new ArgumentException("patternTree cannot be null");
            }
            // x and <ID>, x and y, or x and x; or could be mismatched types
            if (tree is ITerminalNode && patternTree is ITerminalNode)
            {
                ITerminalNode t1             = (ITerminalNode)tree;
                ITerminalNode t2             = (ITerminalNode)patternTree;
                IParseTree    mismatchedNode = null;
                // both are tokens and they have same type
                if (t1.Symbol.Type == t2.Symbol.Type)
                {
                    if (t2.Symbol is TokenTagToken)
                    {
                        // x and <ID>
                        TokenTagToken tokenTagToken = (TokenTagToken)t2.Symbol;
                        // track label->list-of-nodes for both token name and label (if any)

                        labels.Map(tokenTagToken.TokenName, tree);
                        if (tokenTagToken.Label != null)
                        {
                            labels.Map(tokenTagToken.Label, tree);
                        }
                    }
                    else
                    {
                        if (t1.GetText().Equals(t2.GetText(), StringComparison.Ordinal))
                        {
                        }
                        else
                        {
                            // x and x
                            // x and y
                            if (mismatchedNode == null)
                            {
                                mismatchedNode = t1;
                            }
                        }
                    }
                }
                else
                {
                    if (mismatchedNode == null)
                    {
                        mismatchedNode = t1;
                    }
                }
                return(mismatchedNode);
            }
            if (tree is ParserRuleContext && patternTree is ParserRuleContext)
            {
                ParserRuleContext r1             = (ParserRuleContext)tree;
                ParserRuleContext r2             = (ParserRuleContext)patternTree;
                IParseTree        mismatchedNode = null;
                // (expr ...) and <expr>
                RuleTagToken ruleTagToken = GetRuleTagToken(r2);
                if (ruleTagToken != null)
                {
                    if (r1.RuleIndex == r2.RuleIndex)
                    {
                        // track label->list-of-nodes for both rule name and label (if any)
                        labels.Map(ruleTagToken.RuleName, tree);
                        if (ruleTagToken.Label != null)
                        {
                            labels.Map(ruleTagToken.Label, tree);
                        }
                    }
                    else
                    {
                        if (mismatchedNode == null)
                        {
                            mismatchedNode = r1;
                        }
                    }
                    return(mismatchedNode);
                }
                // (expr ...) and (expr ...)
                if (r1.ChildCount != r2.ChildCount)
                {
                    if (mismatchedNode == null)
                    {
                        mismatchedNode = r1;
                    }
                    return(mismatchedNode);
                }
                int n = r1.ChildCount;
                for (int i = 0; i < n; i++)
                {
                    IParseTree childMatch = MatchImpl(r1.GetChild(i), patternTree.GetChild(i), labels);
                    if (childMatch != null)
                    {
                        return(childMatch);
                    }
                }
                return(mismatchedNode);
            }
            // if nodes aren't both tokens or both rule nodes, can't match
            return(tree);
        }