C# (CSharp) IronPython.Compiler Tokenizer.GetNextToken示例

示例#1

        public void TokenAfterFromIsEof()
        {
            FirstTokenIsFrom();
            IPyCompiler.Token token = tokenizer.GetNextToken();

            Assert.AreEqual(IPyCompiler.TokenKind.EndOfFile, token.Kind);
        }

示例#2

文件： ParsePartialFromImportStatementWithTokenizerTestFixture.cs 项目： Rpinski/SharpDevelop

		public void FirstTokenIsFrom()
		{
			string text = "from";
			tokenizer = CreateTokenizer(text);
			IPyCompiler.Token token = tokenizer.GetNextToken();
			Assert.AreEqual(IPyCompiler.TokenKind.KeywordFrom, token.Kind);
		}

示例#3

        public void FirstTokenIsFrom()
        {
            string text = "from";

            tokenizer = CreateTokenizer(text);
            IPyCompiler.Token token = tokenizer.GetNextToken();
            Assert.AreEqual(IPyCompiler.TokenKind.KeywordFrom, token.Kind);
        }

示例#4

文件： ParsePartialFromImportStatementWithTokenizerTestFixture.cs 项目： Bombadil77/SharpDevelop

		IPyCompiler.Token GetToken(string text, int tokenNumber)
		{
			tokenizer = CreateTokenizer(text);
			
			IPyCompiler.Token token = null;
			for (int i = 0; i < tokenNumber; ++i) {
				token = tokenizer.GetNextToken();
			}
			return token;
		}

示例#5

        IPyCompiler.Token GetToken(string text, int tokenNumber)
        {
            tokenizer = CreateTokenizer(text);

            IPyCompiler.Token token = null;
            for (int i = 0; i < tokenNumber; ++i)
            {
                token = tokenizer.GetNextToken();
            }
            return(token);
        }