Esempio n. 1
0
        protected override Token IsMatchImpl(Tokenizer tokenizer)
        {
            var leftOperand = GetIntegers(tokenizer);

            if (leftOperand != null)
            {
                bool   isFloat     = false;
                string numberValue = leftOperand;
                if (tokenizer.Current == ".")
                {
                    tokenizer.Consume();

                    var rightOperand = GetIntegers(tokenizer);

                    if (rightOperand != null)
                    {
                        numberValue += "." + rightOperand;
                        isFloat      = true;
                    }
                }

                if (tokenizer.Current == "e")
                {
                    string rightOperand = "";

                    tokenizer.Consume();

                    if (tokenizer.Current == "-")
                    {
                        rightOperand = "-";
                        tokenizer.Consume();
                    }

                    rightOperand += GetIntegers(tokenizer);

                    if (rightOperand != null)
                    {
                        numberValue += "e" + rightOperand;
                        isFloat      = true;
                    }
                }

                if (!tokenizer.End())
                {
                    if (Char.IsLetter(tokenizer.Current, 0))
                    {
                        return(null);
                    }
                }

                if (isFloat)
                {
                    return(new Token(TokenType.FloatValue, numberValue));
                }

                return(new Token(TokenType.IntValue, leftOperand));
            }

            return(null);
        }
Esempio n. 2
0
        protected override Token IsMatchImpl(Tokenizer tokenizer)
        {
            if (new MatchKeyword(TokenType.Unknown, CommentDelim.Item1).IsMatch(tokenizer) == null)
            {
                return(null);
            }
            else
            {
                tokenizer.Consume();

                while (!tokenizer.End() && new MatchKeyword(TokenType.Unknown, CommentDelim.Item2).IsMatch(tokenizer) == null)
                {
                    tokenizer.Consume();
                }

                if (new MatchKeyword(TokenType.Unknown, CommentDelim.Item2).IsMatch(tokenizer) == null)
                {
                    tokenizer.Consume();
                }

                TokenType tokenType = TokenType.OpenBlockComment;

                if (CommentDelim == LINE_COMMENT)
                {
                    tokenType = TokenType.OpenLineComment;
                }

                return(new Token(tokenType, CommentDelim.Item1));
            }
        }
        protected override Token IsMatchImpl(Tokenizer tokenizer)
        {
            foreach (var character in Match)
            {
                if (tokenizer.Current == character.ToString(CultureInfo.InvariantCulture))
                {
                    tokenizer.Consume();
                }
                else
                {
                    return(null);
                }
            }

            bool found;

            if (!AllowAsSubString)
            {
                var next = tokenizer.Current;

                found = String.IsNullOrWhiteSpace(next) || SpecialCharacters.Any(character => character.Match == next);
            }
            else
            {
                found = true;
            }

            return(found ? new Token(TokenType, Match) : null);
        }
        protected override Token IsMatchImpl(Tokenizer tokenizer)
        {
            var str = new StringBuilder();

            if (tokenizer.Current == StringDelim)
            {
                tokenizer.Consume();

                while (!tokenizer.End() && tokenizer.Current != StringDelim)
                {
                    str.Append(tokenizer.Current);
                    tokenizer.Consume();
                }

                if (tokenizer.End())
                {
                    return(null);
                }

                if (tokenizer.Current == StringDelim)
                {
                    tokenizer.Consume();
                }
            }

            TokenType tokenType;

            if (StringDelim == TIC)
            {
                tokenType = TokenType.CharValue;
                if (str.Length > 1)
                {
                    return(null);
                }
            }
            else
            {
                tokenType = TokenType.StringValue;
            }
            return(str.Length > 0 ? new Token(tokenType, StringDelim + str + StringDelim) : null);
        }
Esempio n. 5
0
        protected override Token IsMatchImpl(Tokenizer tokenizer)
        {
            bool foundWhiteSpace = false;

            while (!tokenizer.End() && String.IsNullOrWhiteSpace(tokenizer.Current))
            {
                foundWhiteSpace = true;

                tokenizer.Consume();
            }

            return(foundWhiteSpace ? new Token(TokenType.WhiteSpace) : null);
        }
Esempio n. 6
0
        private String GetIntegers(Tokenizer tokenizer)
        {
            var regex = new Regex("[0-9]");

            String num = null;

            while (tokenizer.Current != null && regex.IsMatch(tokenizer.Current))
            {
                num += tokenizer.Current;
                tokenizer.Consume();
            }

            return(num != null ? num : null);
        }
Esempio n. 7
0
        protected override Token IsMatchImpl(Tokenizer tokenizer)
        {
            String current = null;

            while (!tokenizer.End() && !String.IsNullOrWhiteSpace(tokenizer.Current) && SpecialCharacters.All(m => m.Match != tokenizer.Current))
            {
                current += tokenizer.Current;
                tokenizer.Consume();
            }

            TokenType tokenType = TokenType.Unknown;

            if (current == null)
            {
                return(null);
            }

            if (Char.IsLetter(current, 0))
            {
                tokenType = TokenType.Identifier;
            }

            return(new Token(tokenType, current));
        }