Ejemplo n.º 1
0
        private Token ParseToken()
        {
            var TokType = TokenType.Error;

            CurrentType_ = LexerStateType.Begin;
            CurrentText_.Reset();

            var TokenLine      = CurrentLine_;
            var TokenLineIndex = CurrentLineIndex_;

            while (!CharsStream_.IsEnd() && CurrentType_ != LexerStateType.End)
            {
                var Ch = CharsStream_.Take();
                CurrentText_.Push(Ch);
                CurrentLineIndex_++;

                switch (CurrentType_)
                {
                    #region Begin
                case LexerStateType.Begin:
                    if (LanguageDescriptor.IsWhitespaceChar(Ch))
                    {
                        TokType      = TokenType.None;
                        CurrentType_ = LexerStateType.End;
                    }
                    else if (LanguageDescriptor.IsEndOfLineChar(Ch))
                    {
                        CurrentLine_++;
                        CurrentLineIndex_ = 0;

                        TokType      = TokenType.None;
                        CurrentType_ = LexerStateType.End;
                    }
                    else if (LanguageDescriptor.IsDelimiterChar(Ch))
                    {
                        TokType      = TokenType.Delimiter;
                        CurrentType_ = LexerStateType.End;
                    }
                    else if (LanguageDescriptor.IsDigitChar(Ch))
                    {
                        CurrentType_ = LexerStateType.Number;
                    }
                    else if (LanguageDescriptor.IsQuoteChar(Ch))
                    {
                        CurrentType_ = LexerStateType.String;
                    }
                    else if (LanguageDescriptor.IsIdentityChar(Ch))
                    {
                        CurrentType_ = LexerStateType.Identity;
                    }
                    break;

                    #endregion
                    #region Number
                case LexerStateType.Number:
                    if (LanguageDescriptor.IsPointChar(Ch))
                    {
                        CurrentType_ = LexerStateType.Float;
                    }
                    else if (!LanguageDescriptor.IsDigitChar(Ch))
                    {
                        CurrentText_.Pop();
                        CharsStream_.Back();

                        if (LanguageDescriptor.IsWhitespaceChar(Ch) ||
                            LanguageDescriptor.IsDelimiterChar(Ch) ||
                            LanguageDescriptor.IsEndOfLineChar(Ch))
                        {
                            TokType = TokenType.Numeric;
                        }
                        else
                        {
                            TokType = TokenType.Error;
                            Logger.Add(CurrentLine_, CurrentLineIndex_, $"unexpected character '{Ch}' in {CurrentText_}");
                        }

                        CurrentType_ = LexerStateType.End;
                    }
                    break;

                    #endregion
                    #region Float
                case LexerStateType.Float:
                    if (!LanguageDescriptor.IsDigitChar(Ch))
                    {
                        CurrentText_.Pop();
                        CharsStream_.Back();

                        if (LanguageDescriptor.IsWhitespaceChar(Ch) ||
                            LanguageDescriptor.IsDelimiterChar(Ch) ||
                            LanguageDescriptor.IsEndOfLineChar(Ch))
                        {
                            TokType = TokenType.Numeric;
                        }
                        else
                        {
                            TokType = TokenType.Error;
                            Logger.Add(CurrentLine_, CurrentLineIndex_, $"unexpected character '{Ch}' in {CurrentText_}");
                        }

                        CurrentType_ = LexerStateType.End;
                    }
                    break;

                    #endregion
                    #region String
                case LexerStateType.String:
                    if (LanguageDescriptor.IsEscapeChar(Ch))
                    {
                        CurrentText_.Pop();
                        CurrentText_.Push(LanguageDescriptor.EscapeChar(CharsStream_.Take()));
                    }
                    else if (LanguageDescriptor.IsQuoteChar(Ch) && CurrentText_.Index(0) == Ch)
                    {
                        CurrentText_.Pop();
                        CurrentText_.Remove(0);

                        TokType      = TokenType.String;
                        CurrentType_ = LexerStateType.End;
                    }
                    else if (LanguageDescriptor.IsEndOfLineChar(Ch))
                    {
                        CurrentText_.Pop();
                        CharsStream_.Back();

                        TokType      = TokenType.Error;
                        CurrentType_ = LexerStateType.End;
                        Logger.Add(CurrentLine_, CurrentLineIndex_, $"unexpected <eof> in {CurrentText_}");
                    }
                    break;

                    #endregion
                    #region Identity
                case LexerStateType.Identity:
                    if (!LanguageDescriptor.IsIdentityChar(Ch) && !LanguageDescriptor.IsDigitChar(Ch))
                    {
                        CurrentText_.Pop();
                        CharsStream_.Back();
                        TokType      = TokenType.Identity;
                        CurrentType_ = LexerStateType.End;
                    }
                    break;
                    #endregion
                }
            }

            var TokCode = CurrentText_.ToString();
            if (LanguageDescriptor.IsBooleanString(TokCode))
            {
                TokType = TokenType.Boolean;
            }
            else if (LanguageDescriptor.IsNullString(TokCode))
            {
                TokType = TokenType.Null;
            }

            return(new Token(TokType, TokCode, TokenLine, TokenLineIndex));
        }
Ejemplo n.º 2
0
        private Token ParseToken()
        {
            var TokType = TokenType.Error;

            CurrentType_ = LexerStateType.Begin;
            CurrentText_.Reset();

            while (!CharStream_.IsEnd() && CurrentType_ != LexerStateType.End)
            {
                var Ch = CharStream_.Take();
                CurrentText_.Push(Ch);

                switch (CurrentType_)
                {
                    #region Begin
                case LexerStateType.Begin:
                    if (LangDesc_.IsWhitespaceChar(Ch))
                    {
                        TokType      = TokenType.None;
                        CurrentType_ = LexerStateType.End;
                    }
                    else if (LangDesc_.IsEndOfLineChar(Ch))
                    {
                        CurrentLine_++;

                        TokType      = TokenType.None;
                        CurrentType_ = LexerStateType.End;
                    }
                    else if (LangDesc_.IsDelimiterChar(Ch))
                    {
                        TokType      = TokenType.Delimiter;
                        CurrentType_ = LexerStateType.End;
                    }
                    else if (LangDesc_.IsDigitChar(Ch))
                    {
                        CurrentType_ = LexerStateType.Integer;
                    }
                    else if (LangDesc_.IsQuoteChar(Ch))
                    {
                        CurrentType_ = LexerStateType.String;
                    }
                    else if (LangDesc_.IsIdentityChar(Ch))
                    {
                        CurrentType_ = LexerStateType.Identity;
                    }
                    else if (LangDesc_.IsOperatorChar(Ch))
                    {
                        CurrentType_ = LexerStateType.Operator;
                    }
                    break;

                    #endregion
                    #region Integer
                case LexerStateType.Integer:
                    if (Ch == '.')
                    {
                        CurrentType_ = LexerStateType.Float;
                    }
                    else if (!LangDesc_.IsDigitChar(Ch))
                    {
                        CurrentText_.Pop();
                        CharStream_.Back();

                        if (LangDesc_.IsWhitespaceChar(Ch) ||
                            LangDesc_.IsOperatorChar(Ch) ||
                            LangDesc_.IsDelimiterChar(Ch) ||
                            LangDesc_.IsEndOfLineChar(Ch))
                        {
                            TokType = TokenType.Numeric;
                        }
                        else
                        {
                            TokType = TokenType.Error;
                        }

                        CurrentType_ = LexerStateType.End;
                    }
                    break;

                    #endregion
                    #region Float
                case LexerStateType.Float:
                    if (!LangDesc_.IsDigitChar(Ch))
                    {
                        CurrentText_.Pop();
                        CharStream_.Back();

                        if (LangDesc_.IsWhitespaceChar(Ch) || LangDesc_.IsDelimiterChar(Ch))
                        {
                            TokType = TokenType.Numeric;
                        }
                        else
                        {
                            TokType = TokenType.Error;
                        }

                        CurrentType_ = LexerStateType.End;
                    }
                    break;

                    #endregion
                    #region String
                case LexerStateType.String:
                    if (Ch == '\\')
                    {
                        CurrentText_.Pop();
                        CurrentText_.Push(CharStream_.Take());
                    }
                    else if (LangDesc_.IsQuoteChar(Ch) && CurrentText_.Index(0) == Ch)
                    {
                        CurrentText_.Pop();
                        CurrentText_.Remove(0);

                        TokType      = TokenType.String;
                        CurrentType_ = LexerStateType.End;
                    }
                    else if (LangDesc_.IsEndOfLineChar(Ch))
                    {
                        CurrentText_.Pop();
                        CharStream_.Back();

                        TokType      = TokenType.Error;
                        CurrentType_ = LexerStateType.End;
                    }
                    break;

                    #endregion
                    #region Identity
                case LexerStateType.Identity:
                    if (!LangDesc_.IsIdentityChar(Ch) && !LangDesc_.IsDigitChar(Ch))
                    {
                        CurrentText_.Pop();
                        CharStream_.Back();

                        var TokCode = CurrentText_.ToString();
                        if (LangDesc_.IsNilString(TokCode))
                        {
                            TokType = TokenType.Nil;
                        }
                        else if (LangDesc_.IsBooleanString(TokCode))
                        {
                            TokType = TokenType.Boolean;
                        }
                        else if (LangDesc_.IsKeywordString(TokCode))
                        {
                            TokType = TokenType.Keyword;
                        }
                        else
                        {
                            TokType = TokenType.Identifier;
                        }
                        CurrentType_ = LexerStateType.End;
                    }
                    break;

                    #endregion
                    #region Operator
                case LexerStateType.Operator:
                    if (!LangDesc_.IsOperatorChar(Ch))
                    {
                        CurrentText_.Pop();
                        CharStream_.Back();

                        if (LangDesc_.IsOperatorString(CurrentText_.ToString()))
                        {
                            TokType = TokenType.Operator;
                        }
                        else
                        {
                            TokType = TokenType.Error;
                        }

                        CurrentType_ = LexerStateType.End;
                    }
                    break;
                    #endregion
                }
            }

            return(new Token(TokType, CurrentText_.ToString(), CurrentLine_));
        }