Esempio n. 1
0
        private DfaState InitToken(char ch)
        {
            if (_tokenText != null && _tokenText.Length > 0)
            {
                _token.Text = _tokenText.ToString();
                _tokens.Add(_token);

                _tokenText = new StringBuilder();
                _token     = new SimpleToken();
            }

            DfaState newState = DfaState.Initial;

            if (IsAlpha(ch))
            {
                if (ch == 'i')
                {
                    newState = DfaState.Id_int1;
                }
                else
                {
                    newState = DfaState.Id;
                }
                _token.Type = TokenType.Identifier;
                _tokenText.Append(ch);
            }
            else if (IsDigit(ch))
            {
                newState    = DfaState.IntLiteral;
                _token.Type = TokenType.IntLiteral;
                _tokenText.Append(ch);
            }
            else if (ch == '>')
            {
                newState    = DfaState.GT;
                _token.Type = TokenType.GT;
                _tokenText.Append(ch);
            }
            else if (ch == '+')
            {
                newState    = DfaState.Plus;
                _token.Type = TokenType.Plus;
                _tokenText.Append(ch);
            }
            else if (ch == '-')
            {
                newState    = DfaState.Minus;
                _token.Type = TokenType.Minus;
                _tokenText.Append(ch);
            }
            else if (ch == '*')
            {
                newState    = DfaState.Star;
                _token.Type = TokenType.Star;
                _tokenText.Append(ch);
            }
            else if (ch == '/')
            {
                newState    = DfaState.Slash;
                _token.Type = TokenType.Slash;
                _tokenText.Append(ch);
            }
            else if (ch == ';')
            {
                newState    = DfaState.SemiColon;
                _token.Type = TokenType.SemiColon;
                _tokenText.Append(ch);
            }
            else if (ch == '(')
            {
                newState    = DfaState.LeftParen;
                _token.Type = TokenType.LeftParen;
                _tokenText.Append(ch);
            }
            else if (ch == ')')
            {
                newState    = DfaState.RightParen;
                _token.Type = TokenType.RightParen;
                _tokenText.Append(ch);
            }
            else if (ch == '=')
            {
                newState    = DfaState.Assignment;
                _token.Type = TokenType.Assignment;
                _tokenText.Append(ch);
            }
            else
            {
                newState = DfaState.Initial;
            }
            return(newState);
        }
Esempio n. 2
0
        public SimpleTokenReader Tokenize(string code)
        {
            _tokens = new List <Token>();
            StringReader reader = new StringReader(code);

            _tokenText = new StringBuilder();
            _token     = new SimpleToken();
            int  ich = 0;
            char ch  = '0';

            DfaState state = DfaState.Initial;

            try
            {
                while ((ich = reader.Read()) != -1)
                {
                    ch = (char)ich;
                    switch (state)
                    {
                    case DfaState.Initial:
                        state = InitToken(ch);
                        break;

                    case DfaState.Id:
                        if (IsAlpha(ch) || IsDigit(ch))
                        {
                            _tokenText.Append(ch);
                        }
                        else
                        {
                            state = InitToken(ch);
                        }
                        break;

                    case DfaState.GT:
                        if (ch == '=')
                        {
                            _token.Type = TokenType.GE;
                            state       = DfaState.GE;
                            _tokenText.Append(ch);
                        }
                        else
                        {
                            state = InitToken(ch);
                        }
                        break;

                    case DfaState.GE:
                    case DfaState.Assignment:
                    case DfaState.Plus:
                    case DfaState.Minus:
                    case DfaState.Star:
                    case DfaState.Slash:
                    case DfaState.SemiColon:
                    case DfaState.LeftParen:
                    case DfaState.RightParen:
                        state = InitToken(ch);
                        break;

                    case DfaState.IntLiteral:
                        if (IsDigit(ch))
                        {
                            _tokenText.Append(ch);
                        }
                        else
                        {
                            state = InitToken(ch);
                        }
                        break;

                    case DfaState.Id_int1:
                        if (ch == 'n')
                        {
                            state = DfaState.Id_int2;
                            _tokenText.Append(ch);
                        }
                        else if (IsDigit(ch) || IsAlpha(ch))
                        {
                            state = DfaState.Id;
                            _tokenText.Append(ch);
                        }
                        else
                        {
                            state = InitToken(ch);
                        }
                        break;

                    case DfaState.Id_int2:
                        if (ch == 't')
                        {
                            state = DfaState.Id_int3;
                            _tokenText.Append(ch);
                        }
                        else if (IsDigit(ch) || IsAlpha(ch))
                        {
                            state = DfaState.Id;
                            _tokenText.Append(ch);
                        }
                        else
                        {
                            state = InitToken(ch);
                        }
                        break;

                    case DfaState.Id_int3:
                        if (IsBlank(ch))
                        {
                            _token.Type = TokenType.Int;
                            state       = InitToken(ch);
                        }
                        else
                        {
                            state = DfaState.Id;
                            _tokenText.Append(ch);
                        }
                        break;

                    default:
                        break;
                    }
                }

                if (_tokenText.Length > 0)
                {
                    InitToken(ch);
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
            }
            return(new SimpleTokenReader(_tokens));
        }