示例#1
0
        public Token NextToken()
        {
            if (peekedWords.Count > 0)
            {
                return(peekedWords.Dequeue());
            }

            Token res = new Token();

            char peekChar = stream.Peek();

            while (stream.PeekEquals("//"))
            {
                stream.DumpUntil('\n');
                stream.Next();
                peekChar = stream.Peek();
            }

            while (peekChar == ' ' || peekChar == '\n' || peekChar == '\r' || peekChar == '\t')
            {
                if (peekChar == '\n')
                {
                    TabIndex    = 0;
                    StartOfLine = true;
                }
                else if (peekChar == '\t' && StartOfLine)
                {
                    TabIndex++;
                }
                stream.Next();
                peekChar = stream.Peek();
            }

            res.row    = stream.Row;
            res.column = stream.Column;

            if (StartOfLine)
            {
                res.type    = TokenTypes.Punctuation_NewLine;
                res.value   = null;
                StartOfLine = false;
            }
            else if (peekChar == 0)
            {
                res.type  = TokenTypes.EOF;
                res.value = null;
            }
            else if (IsPunctuation(peekChar))
            {
                res.type  = punctuationDictionary[stream.Next()];
                res.value = null;
            }
            else if (char.IsDigit(peekChar))
            {
                res.type  = TokenTypes.Number;
                res.value = ReadNumber();
            }
            else if (peekChar == '"')
            {
                res.type  = TokenTypes.String;
                res.value = ReadString();
            }
            else if (IsOperatorStart(peekChar))
            {
                res.type  = operatorDictionary[ReadOperator()];
                res.value = null;
            }
            else if (IsKeyword(stream.PeekOnly(IsKeywordCharacters)))
            {
                res.type  = TokenTypes.Keyword;
                res.value = stream.NextOnly(IsKeywordCharacters);
            }
            else if (IsIdentifierStart(peekChar))
            {
                res.type  = TokenTypes.Identifier;
                res.value = stream.NextOnly(IsIdentifier);
            }
            else
            {
                throw new Exception("Unknown thing..");
            }

            return(res);
        }