Ejemplo n.º 1
0
        private Token ExtractToken()
        {
            SkipWhitespace();

            char current = source.GetCurrentChar();
            Token token;
            if (current == Source.END_OF_FILE)
            {
                TokenBuilder builder = new TokenBuilder();
                token = builder.CreateTokenWithType(TokenType.END_OF_FILE)
                              .AtLine(source.LineNumber)
                              .AtPosition(source.Position)
                              .Build();
            } else if (Char.IsLetter(current))
            {
                token = ExtractWordToken();
            } else if (Char.IsDigit(current))
            {
                token = ExtractNumberToken();
            } else if (current == '\'')
            {
                token = ExtractStringToken();
            } else if (TokenTypeInfo.IsSpecialSymbol(current.ToString()))
            {
                token = ExtractSpecialToken();
            } else
            {
                TokenBuilder builder = new TokenBuilder();
                token = builder.CreateTokenWithType(TokenType.ERROR)
                               .WithLexeme(current.ToString())
                               .WithValue(ErrorCode.INVALID_CHARACTER)
                               .AtLine(source.LineNumber)
                               .AtPosition(source.Position)
                               .Build();
                source.GetNextChar();
            }
            return token;
        }
Ejemplo n.º 2
0
        private Token ExtractSpecialToken()
        {
            char current = source.GetCurrentChar();

            StringBuilder lexeme = new StringBuilder();
            lexeme.Append(current);

            int pos = source.Position;
            int line= source.LineNumber;

            string value = null;
            TokenType type = TokenType.INVALID;

            switch (current) {
                case '+': case '-': case '*': case '/': case ',':
                case ';': case '\'': case '=': case '(': case ')':
                case '[': case ']': case '{': case '}': case '^':
                    source.GetNextChar();
                    break;
                case ':':
                    current = source.GetNextChar(); // consume ':'
                    if (current == '=')
                    {
                        lexeme.Append(current);
                        source.GetNextChar(); // consume '='
                    }
                    break;
                case '<':
                    current = source.GetNextChar();
                    if (current == '=')
                    {
                        lexeme.Append(current);
                        source.GetNextChar();
                    } else if (current == '>')
                    {
                        lexeme.Append(current);
                        source.GetNextChar();
                    }
                    break;
                case '>':
                    current = source.GetNextChar();
                    if (current == '=')
                    {
                        lexeme.Append(current);
                        source.GetNextChar();
                    }
                    break;
                case '.':
                    current = source.GetNextChar();
                    if (current == '.')
                    {
                        lexeme.Append(current);
                        source.GetNextChar();
                    }
                    break;
                default:
                    source.GetNextChar();
                    type = TokenType.ERROR;
                    value = ErrorCode.INVALID_CHARACTER.Message;
                    break;
            }

            if (type == TokenType.INVALID)
            {
                type = TokenTypeInfo.SpecialTypeFromString(lexeme.ToString());
            }

            TokenBuilder builder = new TokenBuilder();
            return builder.CreateTokenWithType(type).WithLexeme(lexeme.ToString())
                          .WithValue(value)
                          .AtPosition(pos)
                          .AtLine(line).Build();
        }
Ejemplo n.º 3
0
        private Token ExtractStringToken()
        {
            int line = source.LineNumber;
            int pos = source.Position;

            StringBuilder lexeme = new StringBuilder();
            StringBuilder value = new StringBuilder();

            char current = source.GetNextChar(); // consume initial quote
            lexeme.Append('\'');

            // deal with string characters
            do
            {
                // replace any whitespace character with a blank
                if (Char.IsWhiteSpace(current))
                {
                    current = ' ';
                }

                if (current != '\'' && current != Source.END_OF_FILE)
                {
                    lexeme.Append(current);
                    value.Append(current);
                    current = source.GetNextChar(); // again consume
                }

                // quote ? each pair of adjacent quotes represent a single quote.
                if (current == '\'')
                {
                    while (current == '\'' && source.PeekNextChar() == '\'')
                    {
                        lexeme.Append("''");
                        value.Append('\'');
                        current = source.GetNextChar();
                        current = source.GetNextChar(); // consume both quotes
                    }
                }
            } while(current != '\'' && current != Source.END_OF_FILE);

            TokenBuilder builder = new TokenBuilder();
            if (current == '\'')
            {
                source.GetNextChar(); // consume final quote
                lexeme.Append('\'');
                builder.CreateTokenWithType(TokenType.STRING).WithValue(value.ToString()).WithLexeme(lexeme.ToString());
            } else
            {
                builder.CreateTokenWithType(TokenType.ERROR).WithValue(ErrorCode.UNEXPECTED_EOF);
            }
            return builder.AtPosition(pos).AtLine(line).Build();
        }
Ejemplo n.º 4
0
        private Token ExtractNumberToken()
        {
            int line = source.LineNumber;
            int pos = source.Position;

            var uval = UnsignedDigits();

            TokenType err = uval.Item1;
            string lexeme = uval.Item2;
            object value  = uval.Item3;
            string whole_digits = uval.Item4;

            if (err == TokenType.ERROR)
            {
                TokenBuilder builder = new TokenBuilder();
                return builder.CreateTokenWithType(err)
                              .WithLexeme(lexeme)
                              .AtPosition(pos)
                              .AtLine(line)
                              .WithValue(value)
                              .Build();
            }

            // assume int
            TokenType type = TokenType.INTEGER;

            // Is there a . ?
            // It could be a decimal point or the start of a .. token
            char current = source.GetCurrentChar();
            bool saw_dot_dot = false;
            string frac_digits = "";
            if (current == '.')
            {
                if (source.PeekNextChar() == '.')
                {
                    saw_dot_dot = true; // it's a ".." token, so don't consume it
                } else
                {
                    type = TokenType.REAL; // decimal point, so token type is REAL.
                    lexeme += current;
                    current = source.GetNextChar(); // consume decimal point

                    // collect fraction part of the number
                    var uval2 = UnsignedDigits();
                    err = uval2.Item1;
                    string frac_lexeme = uval2.Item2;
                    lexeme += frac_lexeme;
                    value = uval2.Item3;
                    frac_digits = uval2.Item4;

                    if (err == TokenType.ERROR)
                    {
                        TokenBuilder builder = new TokenBuilder();
                        return builder.CreateTokenWithType(err)
                                      .WithLexeme(lexeme)
                                      .AtPosition(pos)
                                      .AtLine(line)
                                      .WithValue(value)
                                      .Build();
                    }
                }
            }

            // is there an exponent part ?
            // There cannot be an exponent if we already saw a ".." token.
            current = source.GetCurrentChar();
            char exponent_sign = '+';
            string exp_digits = "";
            if (!saw_dot_dot && (current == 'E' || current == 'e'))
            {
                type = TokenType.REAL;
                lexeme += current;
                current = source.GetNextChar(); // consume 'E' or 'e'

                // exponent sign ?
                if (current == '+' || current == '-')
                {
                    lexeme += current;
                    exponent_sign = current;
                    current = source.GetNextChar();

                }

                // extract the digits of the exponent.
                err = TokenType.INVALID;
                var uval3 = UnsignedDigits();
                err = uval3.Item1;
                exp_digits = uval3.Item2;
                value = uval3.Item3;

                lexeme += exp_digits;
                if (err == TokenType.ERROR)
                {
                     TokenBuilder builder = new TokenBuilder();
                     return builder.CreateTokenWithType(err)
                                   .WithLexeme(lexeme)
                                   .AtPosition(pos)
                                   .AtLine(line)
                                   .WithValue(value)
                                   .Build();
                }
            }

            // compute the value of an integer number token
            if (type == TokenType.INTEGER)
            {
                var uval4 = ComputeIntVal(whole_digits);
                err = uval4.Item1;
                int val = uval4.Item2;
                value = uval4.Item3;
                if (err == TokenType.ERROR)
                {
                     TokenBuilder builder = new TokenBuilder();
                     return builder.CreateTokenWithType(err)
                                   .WithLexeme(lexeme)
                                   .AtPosition(pos)
                                   .AtLine(line)
                                   .WithValue(value)
                                   .Build();
                } else
                {
                    value = val;
                }
            } else if (type == TokenType.REAL)
            {
                var dval1 = ComputeDoubleVal(whole_digits, frac_digits,
                                                exp_digits, exponent_sign);
                err = dval1.Item1;
                double dval = dval1.Item2;
                value = dval1.Item3;
                if (err == TokenType.ERROR)
                {
                    TokenBuilder builder = new TokenBuilder();
                    return builder.CreateTokenWithType(err)
                                  .WithLexeme(lexeme)
                                  .AtPosition(pos)
                                  .AtLine(line)
                                  .WithValue(value)
                                  .Build();
                } else
                {
                    value = dval;
                }
            }

            TokenBuilder builder0 = new TokenBuilder();
            return builder0.CreateTokenWithType(err)
                          .WithLexeme(lexeme)
                          .AtPosition(pos)
                          .AtLine(line)
                          .WithValue(value)
                          .Build();
        }