Пример #1
0
        void CheckForEOF(antlr.IToken token)
        {
            if (antlr.Token.EOF_TYPE != token.Type)
            {
                return;
            }

            EnqueueEOS(token);
            while (CurrentIndentLevel > 0)
            {
                EnqueueDedent();
                _indentStack.Pop();
            }
        }
Пример #2
0
        public void mDQUOTE(bool _createToken)         //throws RecognitionException, CharStreamException, TokenStreamException
        {
            int _ttype; IToken _token = null; int _begin = text.Length;

            _ttype = DQUOTE;

            match('\u0022');
            if (_createToken && (null == _token) && (_ttype != Token.SKIP))
            {
                _token = makeToken(_ttype);
                _token.setText(text.ToString(_begin, text.Length - _begin));
            }
            returnToken_ = _token;
        }
Пример #3
0
        protected void mDIGIT(bool _createToken) //throws RecognitionException, CharStreamException, TokenStreamException
        {
            int _ttype; IToken _token = null; var _begin = text.Length;

            _ttype = DIGIT;

            matchRange('\u0030', '\u0039');
            if (_createToken && (null == _token) && (_ttype != Token.SKIP))
            {
                _token = makeToken(_ttype);
                _token.setText(text.ToString(_begin, text.Length - _begin));
            }
            returnToken_ = _token;
        }
Пример #4
0
    public void mBINOP(bool _createToken)     //throws RecognitionException, CharStreamException, TokenStreamException
    {
        int _ttype; IToken _token = null; int _begin = text.Length;

        _ttype = BINOP;

        switch (cached_LA1)
        {
        case '|':
        {
            match('|');
            break;
        }

        case '&':
        {
            match('&');
            break;
        }

        case '^':
        {
            match('^');
            break;
        }

        default:
            if ((cached_LA1 == '=') && (cached_LA2 == '>'))
            {
                match("=>");
            }
            else if ((cached_LA1 == '=') && (true))
            {
                match('=');
            }
            else
            {
                throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn());
            }
            break;
        }
        if (_createToken && (null == _token) && (_ttype != Token.SKIP))
        {
            _token = makeToken(_ttype);
            _token.setText(text.ToString(_begin, text.Length - _begin));
        }
        returnToken_ = _token;
    }
Пример #5
0
    public void mWS(bool _createToken)     //throws RecognitionException, CharStreamException, TokenStreamException
    {
        int _ttype; IToken _token = null; int _begin = text.Length;

        _ttype = WS;

        {
            switch (cached_LA1)
            {
            case ' ':
            {
                match(' ');
                break;
            }

            case '\t':
            {
                match('\t');
                break;
            }

            case '\n':
            {
                match('\n');
                newline();
                break;
            }

            case '\r':
            {
                match('\r');
                break;
            }

            default:
            {
                throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn());
            }
            }
        }
        _ttype = Token.SKIP;
        if (_createToken && (null == _token) && (_ttype != Token.SKIP))
        {
            _token = makeToken(_ttype);
            _token.setText(text.ToString(_begin, text.Length - _begin));
        }
        returnToken_ = _token;
    }
Пример #6
0
        public void mCRLF(bool _createToken) //throws RecognitionException, CharStreamException, TokenStreamException
        {
            int _ttype; IToken _token = null; var _begin = text.Length;

            _ttype = CRLF;

            mCR(false);
            mLF(false);
            newline();
            if (_createToken && (null == _token) && (_ttype != Token.SKIP))
            {
                _token = makeToken(_ttype);
                _token.setText(text.ToString(_begin, text.Length - _begin));
            }
            returnToken_ = _token;
        }
Пример #7
0
        private Tuple <int, int> CalculateEndpoint(antlr.IToken token, int endLine, int endIndex, int delimiterLength)
        {
            var startIndex = positionMap[token.getLine() - 1][token.getColumn() - 1];
            var startLine  = token.getLine() - 1;

            if (startLine > endLine || startLine == endLine && startIndex > endIndex)
            {
                whitespaces.Add(new TextSpan {
                    iStartLine = endLine, iStartIndex = endIndex, iEndLine = startLine, iEndIndex = startIndex
                });
            }

            endLine  = startLine - 1;
            endIndex = 0;

            var runningIndex = startIndex + delimiterLength;

            foreach (var part in token.getText().Split(new[] { "\r\n" }, StringSplitOptions.None))
            {
                endLine++;
                endIndex     = runningIndex + part.Length;
                runningIndex = 0;
            }
            endIndex += delimiterLength;

            //endIndex = positionMap[endLine][endIndex];

            var cluster = new MappedToken(
                startLine * lineSize + startIndex,
                endIndex - startIndex);

            if (tokenMap.Count > 0 &&
                tokenMap[tokenMap.Count() - 1].Index >= cluster.Index)
            {
                throw new ArgumentException("Token Mapping order");
            }

            tokenMap.Add(cluster);
            return(new Tuple <int, int>(endLine, endIndex));
        }
Пример #8
0
        antlr.IToken BufferUntilNextNonWhiteSpaceToken()
        {
            antlr.IToken token = null;
            while (true)
            {
                token = _istream.nextToken();

                int ttype = token.Type;
                if (antlr.Token.SKIP == ttype)
                {
                    continue;
                }

                if (_wsTokenType == ttype)
                {
                    _buffer.Append(token.getText());
                    continue;
                }

                break;
            }
            return(token);
        }
Пример #9
0
    public void mLIT_CADENA(bool _createToken)     //throws RecognitionException, CharStreamException, TokenStreamException
    {
        int _ttype; IToken _token = null; int _begin = text.Length;

        _ttype = LIT_CADENA;

        int _saveIndex = 0;

        _saveIndex = text.Length;
        match('\'');
        text.Length = _saveIndex;
        {            // ( ... )*
            for (;;)
            {
                if ((tokenSet_2_.member(cached_LA1)))
                {
                    {
                        match(tokenSet_2_);
                    }
                }
                else
                {
                    goto _loop44_breakloop;
                }
            }
            _loop44_breakloop :;
        }            // ( ... )*
        _saveIndex = text.Length;
        match('\'');
        text.Length = _saveIndex;
        if (_createToken && (null == _token) && (_ttype != Token.SKIP))
        {
            _token = makeToken(_ttype);
            _token.setText(text.ToString(_begin, text.Length - _begin));
        }
        returnToken_ = _token;
    }
Пример #10
0
        public void mLINEFOLDER(bool _createToken) //throws RecognitionException, CharStreamException, TokenStreamException
        {
            int _ttype; IToken _token = null; var _begin = text.Length;

            _ttype = LINEFOLDER;

            mCRLF(false);
            {
                switch (cached_LA1)
                {
                case ' ':
                {
                    mSPACE(false);
                    break;
                }

                case '\t':
                {
                    mHTAB(false);
                    break;
                }

                default:
                {
                    throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn());
                }
                }
            }
            _ttype = Token.SKIP;
            if (_createToken && (null == _token) && (_ttype != Token.SKIP))
            {
                _token = makeToken(_ttype);
                _token.setText(text.ToString(_begin, text.Length - _begin));
            }
            returnToken_ = _token;
        }
Пример #11
0
    public void mEMARK(bool _createToken)     //throws RecognitionException, CharStreamException, TokenStreamException
    {
        int _ttype; IToken _token = null; int _begin = text.Length;

        _ttype = EMARK;

        match('!');
        {
            if ((cached_LA1 == '='))
            {
                match('=');
                _ttype = BINOP;
            }
            else
            {
            }
        }
        if (_createToken && (null == _token) && (_ttype != Token.SKIP))
        {
            _token = makeToken(_ttype);
            _token.setText(text.ToString(_begin, text.Length - _begin));
        }
        returnToken_ = _token;
    }
Пример #12
0
 public static SourceLocation ToEndSourceLocation(antlr.IToken token)
 {
     return(new SourceLocation(token.getLine(), token.getColumn() + token.getText().Length - 1));
 }
Пример #13
0
 void ProcessNextNonWhiteSpaceToken(antlr.IToken token)
 {
     _lastNonWsToken = token;
     Enqueue(token);
 }
Пример #14
0
 void Enqueue(antlr.IToken token)
 {
     _pendingTokens.Enqueue(token);
 }
Пример #15
0
    public void mID(bool _createToken)     //throws RecognitionException, CharStreamException, TokenStreamException
    {
        int _ttype; IToken _token = null; int _begin = text.Length;

        _ttype = ID;

        switch (cached_LA1)
        {
        case 'A':
        case 'B':
        case 'C':
        case 'D':
        case 'E':
        case 'F':
        case 'G':
        case 'H':
        case 'I':
        case 'J':
        case 'K':
        case 'L':
        case 'M':
        case 'N':
        case 'O':
        case 'P':
        case 'Q':
        case 'R':
        case 'S':
        case 'T':
        case 'U':
        case 'V':
        case 'W':
        case 'X':
        case 'Y':
        case 'Z':
        case '_':
        case 'a':
        case 'b':
        case 'c':
        case 'd':
        case 'e':
        case 'f':
        case 'g':
        case 'h':
        case 'i':
        case 'j':
        case 'k':
        case 'l':
        case 'm':
        case 'n':
        case 'o':
        case 'p':
        case 'q':
        case 'r':
        case 's':
        case 't':
        case 'u':
        case 'v':
        case 'w':
        case 'x':
        case 'y':
        case 'z':
        {
            {
                switch (cached_LA1)
                {
                case 'a':
                case 'b':
                case 'c':
                case 'd':
                case 'e':
                case 'f':
                case 'g':
                case 'h':
                case 'i':
                case 'j':
                case 'k':
                case 'l':
                case 'm':
                case 'n':
                case 'o':
                case 'p':
                case 'q':
                case 'r':
                case 's':
                case 't':
                case 'u':
                case 'v':
                case 'w':
                case 'x':
                case 'y':
                case 'z':
                {
                    matchRange('a', 'z');
                    break;
                }

                case 'A':
                case 'B':
                case 'C':
                case 'D':
                case 'E':
                case 'F':
                case 'G':
                case 'H':
                case 'I':
                case 'J':
                case 'K':
                case 'L':
                case 'M':
                case 'N':
                case 'O':
                case 'P':
                case 'Q':
                case 'R':
                case 'S':
                case 'T':
                case 'U':
                case 'V':
                case 'W':
                case 'X':
                case 'Y':
                case 'Z':
                {
                    matchRange('A', 'Z');
                    break;
                }

                case '_':
                {
                    match('_');
                    break;
                }

                default:
                {
                    throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn());
                }
                }
            }
            {                // ( ... )*
                for (;;)
                {
                    switch (cached_LA1)
                    {
                    case 'a':
                    case 'b':
                    case 'c':
                    case 'd':
                    case 'e':
                    case 'f':
                    case 'g':
                    case 'h':
                    case 'i':
                    case 'j':
                    case 'k':
                    case 'l':
                    case 'm':
                    case 'n':
                    case 'o':
                    case 'p':
                    case 'q':
                    case 'r':
                    case 's':
                    case 't':
                    case 'u':
                    case 'v':
                    case 'w':
                    case 'x':
                    case 'y':
                    case 'z':
                    {
                        matchRange('a', 'z');
                        break;
                    }

                    case 'A':
                    case 'B':
                    case 'C':
                    case 'D':
                    case 'E':
                    case 'F':
                    case 'G':
                    case 'H':
                    case 'I':
                    case 'J':
                    case 'K':
                    case 'L':
                    case 'M':
                    case 'N':
                    case 'O':
                    case 'P':
                    case 'Q':
                    case 'R':
                    case 'S':
                    case 'T':
                    case 'U':
                    case 'V':
                    case 'W':
                    case 'X':
                    case 'Y':
                    case 'Z':
                    {
                        matchRange('A', 'Z');
                        break;
                    }

                    case '0':
                    case '1':
                    case '2':
                    case '3':
                    case '4':
                    case '5':
                    case '6':
                    case '7':
                    case '8':
                    case '9':
                    {
                        matchRange('0', '9');
                        break;
                    }

                    case '_':
                    {
                        match('_');
                        break;
                    }

                    default:
                    {
                        goto _loop50_breakloop;
                    }
                    }
                }
                _loop50_breakloop :;
            }                // ( ... )*
            break;
        }

        case '{':
        {
            {
                match('{');
                {
                    match(tokenSet_0_);
                }
                match('}');
            }
            break;
        }

        default:
        {
            throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn());
        }
        }
        if (_createToken && (null == _token) && (_ttype != Token.SKIP))
        {
            _token = makeToken(_ttype);
            _token.setText(text.ToString(_begin, text.Length - _begin));
        }
        returnToken_ = _token;
    }
Пример #16
0
        protected void mSPECIAL(bool _createToken) //throws RecognitionException, CharStreamException, TokenStreamException
        {
            int _ttype; IToken _token = null; var _begin = text.Length;

            _ttype = SPECIAL;

            switch (cached_LA1)
            {
            case '!':
            {
                match('\u0021');
                break;
            }

            case '#':
            case '$':
            case '%':
            case '&':
            case '\'':
            case '(':
            case ')':
            case '*':
            case '+':
            {
                matchRange('\u0023', '\u002b');
                break;
            }

            case '<':
            {
                match('\u003c');
                break;
            }

            case '>':
            case '?':
            case '@':
            {
                matchRange('\u003e', '\u0040');
                break;
            }

            case '[':
            {
                match('\u005b');
                break;
            }

            case ']':
            case '^':
            {
                matchRange('\u005d', '\u005e');
                break;
            }

            case '`':
            {
                match('\u0060');
                break;
            }

            case '{':
            case '|':
            case '}':
            case '~':
            {
                matchRange('\u007b', '\u007e');
                break;
            }

            default:
                if (((cached_LA1 >= '\u0080' && cached_LA1 <= '\u00ff')))
                {
                    matchRange('\u0080', '\u00ff');
                }
                else
                {
                    throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn());
                }
                break;
            }
            if (_createToken && (null == _token) && (_ttype != Token.SKIP))
            {
                _token = makeToken(_ttype);
                _token.setText(text.ToString(_begin, text.Length - _begin));
            }
            returnToken_ = _token;
        }
Пример #17
0
 public override void initialize(IToken t)
 {
     base.initialize(t);
     line = t.getLine();
     col  = t.getColumn();
 }
Пример #18
0
 public static SourceLocation ToSourceLocation(antlr.IToken token)
 {
     return(new SourceLocation(token.getLine(), token.getColumn()));
 }
Пример #19
0
 void EnqueueIndent(antlr.IToken prototype)
 {
     _pendingTokens.Enqueue(CreateToken(prototype, _indentTokenType, "<INDENT>"));
 }
Пример #20
0
	private void SetEndSourceLocation(Node node, IToken token)
	{
		node.EndSourceLocation = SourceLocationFactory.ToSourceLocation(token);
	}
Пример #21
0
 public MappedToken(ScanningLexer scanningLexer, antlr.IToken iToken)
 {
     lexer       = scanningLexer;
     this.iToken = iToken;
 }
Пример #22
0
        public void mIANA_TOKEN(bool _createToken) //throws RecognitionException, CharStreamException, TokenStreamException
        {
            int _ttype; IToken _token = null; var _begin = text.Length;

            _ttype = IANA_TOKEN;

            {     // ( ... )+
                var _cnt82 = 0;
                for (;;)
                {
                    switch (cached_LA1)
                    {
                    case 'A':
                    case 'B':
                    case 'C':
                    case 'D':
                    case 'E':
                    case 'F':
                    case 'G':
                    case 'H':
                    case 'I':
                    case 'J':
                    case 'K':
                    case 'L':
                    case 'M':
                    case 'N':
                    case 'O':
                    case 'P':
                    case 'Q':
                    case 'R':
                    case 'S':
                    case 'T':
                    case 'U':
                    case 'V':
                    case 'W':
                    case 'X':
                    case 'Y':
                    case 'Z':
                    case 'a':
                    case 'b':
                    case 'c':
                    case 'd':
                    case 'e':
                    case 'f':
                    case 'g':
                    case 'h':
                    case 'i':
                    case 'j':
                    case 'k':
                    case 'l':
                    case 'm':
                    case 'n':
                    case 'o':
                    case 'p':
                    case 'q':
                    case 'r':
                    case 's':
                    case 't':
                    case 'u':
                    case 'v':
                    case 'w':
                    case 'x':
                    case 'y':
                    case 'z':
                    {
                        mALPHA(false);
                        break;
                    }

                    case '0':
                    case '1':
                    case '2':
                    case '3':
                    case '4':
                    case '5':
                    case '6':
                    case '7':
                    case '8':
                    case '9':
                    {
                        mDIGIT(false);
                        break;
                    }

                    case '-':
                    {
                        mDASH(false);
                        break;
                    }

                    case '_':
                    {
                        mUNDERSCORE(false);
                        break;
                    }

                    default:
                        if ((tokenSet_3_.member(cached_LA1)))
                        {
                            mSPECIAL(false);
                        }
                        else if (((cached_LA1 >= '\u0100' && cached_LA1 <= '\ufffe')))
                        {
                            mUNICODE(false);
                        }
                        else
                        {
                            if (_cnt82 >= 1)
                            {
                                goto _loop82_breakloop;
                            }
                            else
                            {
                                throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn());;
                            }
                        }
                        break;
                    }
                    _cnt82++;
                }
                _loop82_breakloop :;
            }        // ( ... )+

            var s = text.ToString(_begin, text.Length - _begin);
            int val;

            if (int.TryParse(s, out val))
            {
                _ttype = NUMBER;
            }
            else
            {
                switch (s.ToUpper())
                {
                case "BEGIN": _ttype = BEGIN; break;

                case "END": _ttype = END; break;

                case "VCALENDAR": _ttype = VCALENDAR; break;

                default:
                    if (s.Length > 2 && s.Substring(0, 2).Equals("X-"))
                    {
                        _ttype = X_NAME;
                    }
                    break;
                }
            }

            if (_createToken && (null == _token) && (_ttype != Token.SKIP))
            {
                _token = makeToken(_ttype);
                _token.setText(text.ToString(_begin, text.Length - _begin));
            }
            returnToken_ = _token;
        }
Пример #23
0
        public void mCTL(bool _createToken) //throws RecognitionException, CharStreamException, TokenStreamException
        {
            int _ttype; IToken _token = null; var _begin = text.Length;

            _ttype = CTL;

            switch (cached_LA1)
            {
            case '\u0000':
            case '\u0001':
            case '\u0002':
            case '\u0003':
            case '\u0004':
            case '\u0005':
            case '\u0006':
            case '\u0007':
            case '\u0008':
            {
                matchRange('\u0000', '\u0008');
                break;
            }

            case '\u000b':
            case '\u000c':
            case '\r':
            case '\u000e':
            case '\u000f':
            case '\u0010':
            case '\u0011':
            case '\u0012':
            case '\u0013':
            case '\u0014':
            case '\u0015':
            case '\u0016':
            case '\u0017':
            case '\u0018':
            case '\u0019':
            case '\u001a':
            case '\u001b':
            case '\u001c':
            case '\u001d':
            case '\u001e':
            case '\u001f':
            {
                matchRange('\u000b', '\u001F');
                break;
            }

            case '\u007f':
            {
                match('\u007F');
                break;
            }

            default:
            {
                throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn());
            }
            }
            if (_createToken && (null == _token) && (_ttype != Token.SKIP))
            {
                _token = makeToken(_ttype);
                _token.setText(text.ToString(_begin, text.Length - _begin));
            }
            returnToken_ = _token;
        }
Пример #24
0
 void EnqueueEOS(antlr.IToken prototype)
 {
     _pendingTokens.Enqueue(CreateToken(prototype, _eosTokenType, "<EOL>"));
 }
Пример #25
0
		override public IToken nextToken()			//throws TokenStreamException
		{
			IToken theRetToken = null;
tryAgain:
			for (;;)
			{
				IToken _token = null;
				int _ttype = Token.INVALID_TYPE;
				resetText();
				try     // for char stream error handling
				{
					try     // for lexical error handling
					{
						switch ( cached_LA1 )
						{
						case '\t':  case '\n':  case '\r':  case ' ':
						{
							mWS(true);
							theRetToken = returnToken_;
							break;
						}
						case '(':
						{
							mLPAREN(true);
							theRetToken = returnToken_;
							break;
						}
						case ')':
						{
							mRPAREN(true);
							theRetToken = returnToken_;
							break;
						}
						case '*':
						{
							mSTAR(true);
							theRetToken = returnToken_;
							break;
						}
						case '+':
						{
							mPLUS(true);
							theRetToken = returnToken_;
							break;
						}
						case ';':
						{
							mSEMI(true);
							theRetToken = returnToken_;
							break;
						}
						case '0':  case '1':  case '2':  case '3':
						case '4':  case '5':  case '6':  case '7':
						case '8':  case '9':
						{
							mINT(true);
							theRetToken = returnToken_;
							break;
						}
						default:
						{
							if (cached_LA1==EOF_CHAR) { uponEOF(); returnToken_ = makeToken(Token.EOF_TYPE); }
				else {throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn());}
						}
						break; }
						if ( null==returnToken_ ) goto tryAgain; // found SKIP token
						_ttype = returnToken_.Type;
						_ttype = testLiteralsTable(_ttype);
						returnToken_.Type = _ttype;
						return returnToken_;
					}
					catch (RecognitionException e) {
							throw new TokenStreamRecognitionException(e);
					}
				}
				catch (CharStreamException cse) {
					if ( cse is CharStreamIOException ) {
						throw new TokenStreamIOException(((CharStreamIOException)cse).io);
					}
					else {
						throw new TokenStreamException(cse.Message);
					}
				}
			}
		}
Пример #26
0
 public static LexicalInfo ToLexicalInfo(antlr.IToken token)
 {
     return(new LexicalInfo(token.getFilename(), token.getLine(), token.getColumn()));
 }
Пример #27
0
 public ASTNode(IToken tok) : base(tok)
 {
 }
Пример #28
0
        public static SourceLocation ToEndSourceLocation(antlr.IToken token)
        {
            string text = token.getText() ?? "";

            return(new SourceLocation(token.getLine(), token.getColumn() + text.Length - 1));
        }
Пример #29
0
    override public IToken nextToken()                                  //throws TokenStreamException
    {
        IToken theRetToken = null;

tryAgain:
        for (;;)
        {
            IToken _token = null;
            int    _ttype = Token.INVALID_TYPE;
            resetText();
            try                         // for char stream error handling
            {
                try                     // for lexical error handling
                {
                    switch (cached_LA1)
                    {
                    case ',':
                    {
                        mCOMMA(true);
                        theRetToken = returnToken_;
                        break;
                    }

                    case ';':
                    {
                        mSEMI(true);
                        theRetToken = returnToken_;
                        break;
                    }

                    case '(':
                    {
                        mLPAREN(true);
                        theRetToken = returnToken_;
                        break;
                    }

                    case ')':
                    {
                        mRPAREN(true);
                        theRetToken = returnToken_;
                        break;
                    }

                    case '?':
                    {
                        mQMARK(true);
                        theRetToken = returnToken_;
                        break;
                    }

                    case '!':
                    {
                        mEMARK(true);
                        theRetToken = returnToken_;
                        break;
                    }

                    case 'A':
                    case 'B':
                    case 'C':
                    case 'D':
                    case 'E':
                    case 'F':
                    case 'G':
                    case 'H':
                    case 'I':
                    case 'J':
                    case 'K':
                    case 'L':
                    case 'M':
                    case 'N':
                    case 'O':
                    case 'P':
                    case 'Q':
                    case 'R':
                    case 'S':
                    case 'T':
                    case 'U':
                    case 'V':
                    case 'W':
                    case 'X':
                    case 'Y':
                    case 'Z':
                    case '_':
                    case 'a':
                    case 'b':
                    case 'c':
                    case 'd':
                    case 'e':
                    case 'f':
                    case 'g':
                    case 'h':
                    case 'i':
                    case 'j':
                    case 'k':
                    case 'l':
                    case 'm':
                    case 'n':
                    case 'o':
                    case 'p':
                    case 'q':
                    case 'r':
                    case 's':
                    case 't':
                    case 'u':
                    case 'v':
                    case 'w':
                    case 'x':
                    case 'y':
                    case 'z':
                    case '{':
                    {
                        mID(true);
                        theRetToken = returnToken_;
                        break;
                    }

                    case '&':
                    case '=':
                    case '^':
                    case '|':
                    {
                        mBINOP(true);
                        theRetToken = returnToken_;
                        break;
                    }

                    case '0':
                    case '1':
                    {
                        mCONST(true);
                        theRetToken = returnToken_;
                        break;
                    }

                    case '\t':
                    case '\n':
                    case '\r':
                    case ' ':
                    {
                        mWS(true);
                        theRetToken = returnToken_;
                        break;
                    }

                    default:
                        if ((cached_LA1 == ':') && (cached_LA2 == '='))
                        {
                            mASSIGN(true);
                            theRetToken = returnToken_;
                        }
                        else if ((cached_LA1 == ':') && (true))
                        {
                            mCOLON(true);
                            theRetToken = returnToken_;
                        }
                        else
                        {
                            if (cached_LA1 == EOF_CHAR)
                            {
                                uponEOF(); returnToken_ = makeToken(Token.EOF_TYPE);
                            }
                            else
                            {
                                throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn());
                            }
                        }
                        break;
                    }
                    if (null == returnToken_)
                    {
                        goto tryAgain;                                                   // found SKIP token
                    }
                    _ttype            = returnToken_.Type;
                    _ttype            = testLiteralsTable(_ttype);
                    returnToken_.Type = _ttype;
                    return(returnToken_);
                }
                catch (RecognitionException e) {
                    throw new TokenStreamRecognitionException(e);
                }
            }
            catch (CharStreamException cse) {
                if (cse is CharStreamIOException)
                {
                    throw new TokenStreamIOException(((CharStreamIOException)cse).io);
                }
                else
                {
                    throw new TokenStreamException(cse.Message);
                }
            }
        }
    }
Пример #30
0
        override public IToken nextToken()                              //throws TokenStreamException
        {
            IToken theRetToken = null;

tryAgain:
            for (;;)
            {
                IToken _token = null;
                var    _ttype = Token.INVALID_TYPE;
                resetText();
                try                     // for char stream error handling
                {
                    try                 // for lexical error handling
                    {
                        switch (cached_LA1)
                        {
                        case '\n':
                        {
                            mLF(true);
                            theRetToken = returnToken_;
                            break;
                        }

                        case ' ':
                        {
                            mSPACE(true);
                            theRetToken = returnToken_;
                            break;
                        }

                        case '\t':
                        {
                            mHTAB(true);
                            theRetToken = returnToken_;
                            break;
                        }

                        case ':':
                        {
                            mCOLON(true);
                            theRetToken = returnToken_;
                            break;
                        }

                        case ';':
                        {
                            mSEMICOLON(true);
                            theRetToken = returnToken_;
                            break;
                        }

                        case ',':
                        {
                            mCOMMA(true);
                            theRetToken = returnToken_;
                            break;
                        }

                        case '.':
                        {
                            mDOT(true);
                            theRetToken = returnToken_;
                            break;
                        }

                        case '=':
                        {
                            mEQUAL(true);
                            theRetToken = returnToken_;
                            break;
                        }

                        case '/':
                        {
                            mSLASH(true);
                            theRetToken = returnToken_;
                            break;
                        }

                        case '"':
                        {
                            mDQUOTE(true);
                            theRetToken = returnToken_;
                            break;
                        }

                        default:
                            if ((cached_LA1 == '\r') && (cached_LA2 == '\n') && (LA(3) == '\t' || LA(3) == ' '))
                            {
                                mLINEFOLDER(true);
                                theRetToken = returnToken_;
                            }
                            else if ((cached_LA1 == '\r') && (cached_LA2 == '\n') && (true))
                            {
                                mCRLF(true);
                                theRetToken = returnToken_;
                            }
                            else if ((cached_LA1 == '\\') && (tokenSet_0_.member(cached_LA2)))
                            {
                                mESCAPED_CHAR(true);
                                theRetToken = returnToken_;
                            }
                            else if ((cached_LA1 == '\\') && (true))
                            {
                                mBACKSLASH(true);
                                theRetToken = returnToken_;
                            }
                            else if ((tokenSet_1_.member(cached_LA1)) && (true))
                            {
                                mCTL(true);
                                theRetToken = returnToken_;
                            }
                            else if ((tokenSet_2_.member(cached_LA1)))
                            {
                                mIANA_TOKEN(true);
                                theRetToken = returnToken_;
                            }
                            else
                            {
                                if (cached_LA1 == EOF_CHAR)
                                {
                                    uponEOF(); returnToken_ = makeToken(Token.EOF_TYPE);
                                }
                                else
                                {
                                    throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn());
                                }
                            }
                            break;
                        }
                        if (null == returnToken_)
                        {
                            goto tryAgain;                                               // found SKIP token
                        }
                        _ttype            = returnToken_.Type;
                        _ttype            = testLiteralsTable(_ttype);
                        returnToken_.Type = _ttype;
                        return(returnToken_);
                    }
                    catch (RecognitionException e) {
                        throw new TokenStreamRecognitionException(e);
                    }
                }
                catch (CharStreamException cse) {
                    if (cse is CharStreamIOException)
                    {
                        throw new TokenStreamIOException(((CharStreamIOException)cse).io);
                    }
                    else
                    {
                        throw new TokenStreamException(cse.Message);
                    }
                }
            }
        }
Пример #31
0
        protected void mALPHA(bool _createToken) //throws RecognitionException, CharStreamException, TokenStreamException
        {
            int _ttype; IToken _token = null; var _begin = text.Length;

            _ttype = ALPHA;

            switch (cached_LA1)
            {
            case 'A':
            case 'B':
            case 'C':
            case 'D':
            case 'E':
            case 'F':
            case 'G':
            case 'H':
            case 'I':
            case 'J':
            case 'K':
            case 'L':
            case 'M':
            case 'N':
            case 'O':
            case 'P':
            case 'Q':
            case 'R':
            case 'S':
            case 'T':
            case 'U':
            case 'V':
            case 'W':
            case 'X':
            case 'Y':
            case 'Z':
            {
                matchRange('\u0041', '\u005a');
                break;
            }

            case 'a':
            case 'b':
            case 'c':
            case 'd':
            case 'e':
            case 'f':
            case 'g':
            case 'h':
            case 'i':
            case 'j':
            case 'k':
            case 'l':
            case 'm':
            case 'n':
            case 'o':
            case 'p':
            case 'q':
            case 'r':
            case 's':
            case 't':
            case 'u':
            case 'v':
            case 'w':
            case 'x':
            case 'y':
            case 'z':
            {
                matchRange('\u0061', '\u007a');
                break;
            }

            default:
            {
                throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn());
            }
            }
            if (_createToken && (null == _token) && (_ttype != Token.SKIP))
            {
                _token = makeToken(_ttype);
                _token.setText(text.ToString(_begin, text.Length - _begin));
            }
            returnToken_ = _token;
        }
Пример #32
0
	private LexicalInfo ToLexicalInfo(IToken token)
	{
		return SourceLocationFactory.ToLexicalInfo(token);
	}
Пример #33
0
 void ProcessNextNonWhiteSpaceToken(antlr.IToken token)
 {
     _lastNonWsToken = token;
     Enqueue(token);
 }
Пример #34
0
	private MemberReferenceExpression MemberReferenceForToken(Expression target, IToken memberName)
	{
		MemberReferenceExpression mre = new MemberReferenceExpression(ToLexicalInfo(memberName));
		mre.Target = target;
		mre.Name = memberName.getText();
		return mre;	
	}
Пример #35
0
 public MappedToken(ScanningLexer scanningLexer, antlr.IToken iToken)
 {
     lexer = scanningLexer;
     this.iToken = iToken;
 }