NextToken() public method

public NextToken ( ) : Token
return Token
Example #1
0
        public void ParseDotNetMethodMethodWithParenthesis()
        {
            Lexer tokenizer = new Lexer("(anObject !nativeMethod)");
            Token token;

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("(", token.Value);
            Assert.AreEqual(TokenType.Punctuation, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("anObject", token.Value);
            Assert.AreEqual(TokenType.Name, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("!nativeMethod", token.Value);
            Assert.AreEqual(TokenType.Name, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual(")", token.Value);
            Assert.AreEqual(TokenType.Punctuation, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNull(token);
        }
Example #2
0
        public void ParseDotNetMethodMethod()
        {
            Lexer tokenizer = new Lexer("anObject !nativeMethod");
            Token token;

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("anObject", token.Value);
            Assert.AreEqual(TokenType.Name, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("!nativeMethod", token.Value);
            Assert.AreEqual(TokenType.Name, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNull(token);
        }
Example #3
0
        public void ParseDotNetObjectAndMethod()
        {
            Lexer tokenizer = new Lexer("@System.IO.FileInfo !new: 'FooBar.txt'");
            Token token;

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("@System.IO.FileInfo", token.Value);
            Assert.AreEqual(TokenType.Name, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("!new:", token.Value);
            Assert.AreEqual(TokenType.Name, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("FooBar.txt", token.Value);
            Assert.AreEqual(TokenType.String, token.Type);
        }
Example #4
0
 public void ProcessCollectionStart()
 {
     string punct = "#(";
     Lexer tokenizer = new Lexer(punct);
     Token token;
     token = tokenizer.NextToken();
     Assert.IsNotNull(token);
     Assert.AreEqual(TokenType.Punctuation, token.Type);
     Assert.AreEqual("#(", token.Value);
 }
Example #5
0
 public void SkipComment()
 {
     Lexer tokenizer = new Lexer("\"This is a comment\"");
     Assert.IsNull(tokenizer.NextToken());
 }
Example #6
0
        public void ProcessTwoSymbols()
        {
            Lexer tokenizer = new Lexer("#aSymbol #anotherSymbol");
            Token token;

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("aSymbol", token.Value);
            Assert.AreEqual(TokenType.Symbol, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("anotherSymbol", token.Value);
            Assert.AreEqual(TokenType.Symbol, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNull(token);
        }
Example #7
0
        public void ProcessTokenAndString()
        {
            Lexer tokenizer = new Lexer("token 'string'");
            Token token;

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("token", token.Value);
            Assert.AreEqual(TokenType.Name, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("string", token.Value);
            Assert.AreEqual(TokenType.String, token.Type);
        }
Example #8
0
        public void ProcessSpecialNameAndParenthesis()
        {
            Lexer tokenizer = new Lexer("@System.IO.FileInfo)");

            Token token;

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("@System.IO.FileInfo", token.Value);
            Assert.AreEqual(TokenType.Name, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual(")", token.Value);
            Assert.AreEqual(TokenType.Punctuation, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNull(token);
        }
Example #9
0
        public void ProcessReal()
        {
            Lexer tokenizer = new Lexer("10.234");
            Token token;

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("10.234", token.Value);
            Assert.AreEqual(TokenType.Real, token.Type);
        }
Example #10
0
        public void ProcessIntegerNumberWithRadix()
        {
            Lexer tokenizer = new Lexer("16rFF");
            Token token;

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("16rFF", token.Value);
            Assert.AreEqual(TokenType.Integer, token.Type);

            Assert.IsNull(tokenizer.NextToken());
        }
Example #11
0
        public void ProcessInteger()
        {
            Lexer tokenizer = new Lexer("10");
            Token token;

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("10", token.Value);
            Assert.AreEqual(TokenType.Integer, token.Type);
        }
Example #12
0
        public void ProcessEnclosedSymbol()
        {
            Lexer tokenizer = new Lexer("#{Module.Submodule}");
            Token token;

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("Module.Submodule", token.Value);
            Assert.AreEqual(TokenType.Symbol, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNull(token);
        }
Example #13
0
 public void ProcessEmptyString()
 {
     Lexer tokenizer = new Lexer(string.Empty);
     Assert.IsNull(tokenizer.NextToken());
 }
Example #14
0
        public void ProcessDoubleDottedName()
        {
            Lexer tokenizer = new Lexer("Smalltalk.MyPackage.MySubpackage");

            Token token = tokenizer.NextToken();

            Assert.IsNotNull(token);
            Assert.AreEqual("Smalltalk.MyPackage.MySubpackage", token.Value);
            Assert.AreEqual(TokenType.DottedName, token.Type);

            Assert.IsNull(tokenizer.NextToken());
        }
Example #15
0
        public void ProcessComplexSymbol()
        {
            Lexer tokenizer = new Lexer("#aSymbol:with:many>chars");
            Token token;

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("aSymbol:with:many>chars", token.Value);
            Assert.AreEqual(TokenType.Symbol, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNull(token);
        }
Example #16
0
        public void ProcessPunctuations()
        {
            string punct = "().|[];";
            Lexer tokenizer = new Lexer(punct);
            Token token;

            for (int k = 0; k < punct.Length; k++)
            {
                token = tokenizer.NextToken();
                Assert.IsNotNull(token);
                Assert.AreEqual(punct[k], token.Value[0]);
                Assert.AreEqual(1, token.Value.Length);
                Assert.AreEqual(TokenType.Punctuation, token.Type);
            }
        }
Example #17
0
        public void ProcessQuotedSymbol()
        {
            Lexer tokenizer = new Lexer("#'hello world'");
            Token token;

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("hello world", token.Value);
            Assert.AreEqual(TokenType.Symbol, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNull(token);
        }
Example #18
0
        public void ProcessNotClosedString()
        {
            Lexer tokenizer = new Lexer("'string");
            Token token;

            token = tokenizer.NextToken();
        }
Example #19
0
        public void ProcessReturnNegativeNumber()
        {
            Lexer tokenizer = new Lexer("^-1");
            Token token;

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("^", token.Value);
            Assert.AreEqual(TokenType.Operator, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("-", token.Value);
            Assert.AreEqual(TokenType.Operator, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("1", token.Value);
            Assert.AreEqual(TokenType.Integer, token.Type);

            Assert.IsNull(tokenizer.NextToken());
        }
Example #20
0
        public void ProcessNotDottedNameSecondWordIsLower()
        {
            Lexer tokenizer = new Lexer("Smalltalk.myPackage");

            Token token = tokenizer.NextToken();

            Assert.IsNotNull(token);
            Assert.AreEqual("Smalltalk", token.Value);
            Assert.AreEqual(TokenType.Name, token.Type);

            token = tokenizer.NextToken();

            Assert.IsNotNull(token);
            Assert.AreEqual(".", token.Value);
            Assert.AreEqual(TokenType.Punctuation, token.Type);

            token = tokenizer.NextToken();

            Assert.IsNotNull(token);
            Assert.AreEqual("myPackage", token.Value);
            Assert.AreEqual(TokenType.Name, token.Type);

            Assert.IsNull(tokenizer.NextToken());
        }
Example #21
0
        public void ProcessStringWithEscapedDelimiters()
        {
            Lexer tokenizer = new Lexer("'[''do nothing'']'");
            Token token;

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("['do nothing']", token.Value);
            Assert.AreEqual(TokenType.String, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNull(token);
        }
Example #22
0
        public void ProcessOneTokenWithSpacesAndComment()
        {
            Lexer tokenizer = new Lexer(" \"This is a token \" token \"This another comment\"");
            Token token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("token", token.Value);
            Assert.AreEqual(TokenType.Name, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNull(token);
        }
Example #23
0
        public void ProcessTwoNames()
        {
            Lexer tokenizer = new Lexer("@self");

            Token token;

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("@", token.Value);
            Assert.AreEqual(TokenType.Operator, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("self", token.Value);
            Assert.AreEqual(TokenType.Name, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNull(token);
        }
Example #24
0
        public void ProcessOperators()
        {
            string opers = "== ~= ~~ >= <= := ^ < > : = - + * / &";

            string[] opers2 = opers.Split(' ');

            Lexer tokenizer = new Lexer(opers);
            Token token;

            for (int k = 0; k < opers2.Length; k++)
            {
                token = tokenizer.NextToken();
                Assert.IsNotNull(token);
                Assert.AreEqual(opers2[k], token.Value);
                Assert.AreEqual(TokenType.Operator, token.Type);
            }

            Assert.IsNull(tokenizer.NextToken());
        }
Example #25
0
        public void ProcessTwoTokens()
        {
            Lexer tokenizer = new Lexer("token1 token2");
            Token token;

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("token1", token.Value);
            Assert.AreEqual(TokenType.Name, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("token2", token.Value);
            Assert.AreEqual(TokenType.Name, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNull(token);
        }
Example #26
0
        public void ProcessParameter()
        {
            Lexer tokenizer = new Lexer(":x");
            Token token;

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual(TokenType.Parameter, token.Type);
            Assert.AreEqual("x", token.Value);

            token = tokenizer.NextToken();
            Assert.IsNull(token);
        }
Example #27
0
 public void SkipMultiLineComment()
 {
     Lexer tokenizer = new Lexer("\"This is a \n a multi-line\ncomment\"");
     Assert.IsNull(tokenizer.NextToken());
 }
Example #28
0
        public void ProcessPunctuation()
        {
            Lexer tokenizer = new Lexer(".");
            Token token;

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual(".", token.Value);
            Assert.AreEqual(TokenType.Punctuation, token.Type);
        }
Example #29
0
        public void ProcessAtOperator()
        {
            Lexer tokenizer = new Lexer("@");
            Token token;

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("@", token.Value);
            Assert.AreEqual(TokenType.Operator, token.Type);
        }
Example #30
0
        public void ProcessCharacter()
        {
            Lexer tokenizer = new Lexer("$+");
            Token token;

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("+", token.Value);
            Assert.AreEqual(TokenType.Character, token.Type);
        }