Ejemplo n.º 1
0
        private void AddTokenAndResetTemp(TokenType type)
        {
            if (type == TokenType.Empty)
            {
                if (LexerHelper.IsCorrectIdentifier(token))
                {
                    type = TokenType.Variable;
                }
            }

            Token t = new Token(type, token, tokenStart);

            tokens.Add(t);
            token      = "";
            tokenStart = pointer;
        }
Ejemplo n.º 2
0
        private bool IsEndOfToken()
        {
            if (LexerHelper.IsTokenSeparator(currChar))
            {
                return(true);
            }
            if (LexerHelper.IsCorrectIdentifier(token) != LexerHelper.IsCorrectIdentifier(currChar))
            {
                return(true);
            }
            if (LexerHelper.IsNumeric(token) != LexerHelper.IsNumeric(currChar))
            {
                return(true);
            }
            if (GetTokenType(token + currChar) != TokenType.Operation &&
                GetTokenType(token) == TokenType.Operation &&
                GetTokenType(currChar) == TokenType.Operation)
            {
                return(true);
            }

            return(false);
        }
Ejemplo n.º 3
0
        public List <Token> SplitToTokenList(string scriptText)
        {
            scriptText += ";";

            List <Token> tokens = new List <Token>();
            string       token  = "";

            bool isStringValue = false;
            bool isLineComment = false;

            int index       = 0;
            int globalIndex = 0;

            foreach (char s in scriptText)
            {
                globalIndex++;

                if (s == '\r' || s == '\n')
                {
                    if (isLineComment)
                    {
                        isLineComment = false;
                    }
                    continue;
                }

                if (isLineComment)
                {
                    continue;
                }

                if (token.Length == 0 && !isStringValue && s != '"')
                {
                    index  = globalIndex - 1;
                    token += s;
                    continue;
                }

                if (isStringValue)
                {
                }
                else if (s == '/' && token.Length > 0 && token == "/")
                {
                    ResetTemp();
                    isLineComment = true;
                    continue;
                }
                else if (LexerHelper.IsTokenSeparator(token))
                {
                    AddTokenAndResetTemp(token, index);
                }
                else if (LexerHelper.IsTokenSeparator(s))
                {
                    AddTokenAndResetTemp(token, index);
                }
                else if (GetTokenType(token) == TokenType.Operation &&
                         GetTokenType(token + s) != TokenType.Operation)
                {
                    AddTokenAndResetTemp(token, index);
                }
                else if (int.TryParse(token, out _) && !int.TryParse(token + s, out _))
                {
                    AddTokenAndResetTemp(token, index);
                }
                else if (LexerHelper.IsCorrectIdentifier(token) && !LexerHelper.IsCorrectIdentifier(token + s))
                {
                    AddTokenAndResetTemp(token, index);
                }

                if (s == '"')
                {
                    if (isStringValue)
                    {
                        if (token.Length > 0 && token.Last() == '\\')
                        {
                            token  = token.Remove(token.Length - 1, 1);
                            token += '"';
                            continue;
                        }

                        isStringValue = false;

                        AddTokenByTypeAndResetTemp(TokenType.Text, token, index);
                        continue;
                    }
                    else
                    {
                        isStringValue = true;
                        continue;
                    }
                }

                token += s;
            }

            return(tokens);

            void AddTokenAndResetTemp(string tokenText, int textIndex)
            {
                TokenType tokenType = GetTokenType(tokenText);

                if (tokenType != TokenType.Separator)
                {
                    if (tokenType == TokenType.Empty)
                    {
                        tokenType = TokenType.Variable;
                    }

                    tokens.Add(new Token(tokenType, tokenText, textIndex));
                }

                ResetTemp();
            }

            void AddTokenByTypeAndResetTemp(TokenType tokenType, string tokenText, int textIndex)
            {
                tokens.Add(new Token(tokenType, tokenText, textIndex));
                ResetTemp();
            }

            void ResetTemp()
            {
                token = "";
                index = globalIndex - 1;
            }
        }