Exemplo n.º 1
0
        public List<Token> Tokenize(string text)
        {
            List<Token> tokens = new List<Token>();
            Token currentToken = null;
            char currentChar;

            for (int i = 0; i < text.Length; i++)
            {
                currentChar = text[i];

                if (WhitespaceChars.Contains(currentChar)) // Whitespace
                {
                    CheckIdentifier(currentToken);

                    if (KeepWhitespace)
                    {
                        currentToken = new Token(TokenType.Whitespace, currentChar.ToString(), i);
                        tokens.Add(currentToken);
                    }
                    else
                    {
                        currentToken = new Token();
                    }
                }
                else if (SymbolChars.Contains(currentChar)) // Symbol
                {
                    CheckIdentifier(currentToken);

                    currentToken = new Token(TokenType.Symbol, currentChar.ToString(), i);
                    tokens.Add(currentToken);
                }
                else if (LiteralDelimiters.Contains(currentChar)) // Literal
                {
                    CheckIdentifier(currentToken);

                    currentToken = new Token(TokenType.Literal, currentChar.ToString(), i);
                    tokens.Add(currentToken);

                    char delimeter = currentChar;

                    for (i++; i < text.Length; i++)
                    {
                        currentChar = text[i];
                        currentToken.Text += currentChar;

                        if (currentChar == delimeter && !IsEscaped(currentToken.Text))
                        {
                            if (AutoParseLiteral)
                            {
                                currentToken.Text = ParseString(currentToken.Text);
                            }

                            break;
                        }
                    }
                }
                else // Identifier, Numeric, Keyword
                {
                    if (currentToken != null && currentToken.Type == TokenType.Identifier)
                    {
                        currentToken.Text += currentChar;
                    }
                    else
                    {
                        currentToken = new Token(TokenType.Identifier, currentChar.ToString(), i);
                        tokens.Add(currentToken);
                    }

                    if (i + 1 >= text.Length) // EOF
                    {
                        CheckIdentifier(currentToken);
                    }
                }
            }

            return tokens;
        }
Exemplo n.º 2
0
        private void CheckIdentifier(Token token)
        {
            if (token != null && token.Type == TokenType.Identifier)
            {
                double result;

                if (double.TryParse(token.Text, out result))
                {
                    token.Type = TokenType.Numeric;
                }
                else if (Keywords.Contains(token.Text))
                {
                    token.Type = TokenType.Keyword;
                }
            }
        }
Exemplo n.º 3
0
        private void CheckIdentifier(Token token)
        {
            if (token != null && token.Type == TokenType.Identifier)
            {
                double result;

                if (double.TryParse(token.Text, out result))
                {
                    token.Type = TokenType.Numeric;

                    if (tokens.Count > 1)
                    {
                        Token previousToken = tokens[tokens.Count - 2];

                        if (previousToken != null && previousToken.Type == TokenType.Symbol && previousToken.Text.Equals("-", StringComparison.InvariantCulture))
                        {
                            tokens.Remove(previousToken);
                            token.Position--;
                            token.Text = "-" + token.Text;
                        }
                    }
                }
                else if (Keywords.Contains(token.Text))
                {
                    token.Type = TokenType.Keyword;
                }
            }
        }