Splits an input string into a sequence of tokens used during parsing.
示例#1
0
 public void Keyword_ShouldNotEndWithDot() {
     ExpressionTokenizer et = new ExpressionTokenizer();
     try {
         et.InitTokenizer("abc.");
         Assert.Fail();
     } catch (ExpressionParseException) {
     }
 }
示例#2
0
        public void CheckSyntax(string s) {
            _tokenizer = new ExpressionTokenizer();
            _evalMode = EvalMode.ParseOnly;

            _tokenizer.InitTokenizer(s);
            ParseExpression();
            if (_tokenizer.CurrentToken != ExpressionTokenizer.TokenType.EOF) {
                throw BuildParseError("Unexpected token at the end of expression", _tokenizer.CurrentPosition);
            }
        }
示例#3
0
        public object Evaluate(string s) {
            _tokenizer = new ExpressionTokenizer();
            _evalMode = EvalMode.Evaluate;

            _tokenizer.InitTokenizer(s);
            object val = ParseExpression();
            if (_tokenizer.CurrentToken != ExpressionTokenizer.TokenType.EOF) {
                throw BuildParseError("Unexpected token at the end of expression", _tokenizer.CurrentPosition);
            }
            return val;
        }
示例#4
0
        public void Keyword() {
            string [] identifiers = new string[] {
                "a.d",
                "a-d",
                "a.d",
                "a\\d",
                "a_d",
                "_ad",
                "ad5"
                };

            for (int i = 0; i < identifiers.Length; i++) {
                string identifier = identifiers[i];

                ExpressionTokenizer et = new ExpressionTokenizer();
                et.InitTokenizer(identifier);
                Assert.AreEqual (identifier, et.TokenText, "#A1");
                Assert.AreEqual (ExpressionTokenizer.TokenType.Keyword, et.CurrentToken, "#A2:" + identifier);
                et.GetNextToken();
                Assert.AreEqual (identifier, et.TokenText, "#B1");
                Assert.AreEqual (ExpressionTokenizer.TokenType.EOF, et.CurrentToken, "#B2:" + identifier);
            }
        }
        /// <summary>
        /// Evaluates the given expression string and returns the result
        /// </summary>
        /// <param name="input"></param>
        /// <param name="location"></param>
        /// <param name="state"></param>
        /// <param name="visiting"></param>
        /// <returns></returns>
        private string EvaluateEmbeddedExpressions(string input, Location location, Hashtable state, Stack visiting)
        {
            if (input == null)
            {
                return(null);
            }

            if (input.IndexOf('$') < 0)
            {
                return(input);
            }

            try
            {
                StringBuilder output = new StringBuilder(input.Length);

                ExpressionTokenizer tokenizer = new ExpressionTokenizer();
                ExpressionEvaluator eval      = new ExpressionEvaluator(this.project, this, state, visiting, this.callStack);

                tokenizer.IgnoreWhitespace    = false;
                tokenizer.SingleCharacterMode = true;
                tokenizer.InitTokenizer(input);

                while (tokenizer.CurrentToken != ExpressionTokenizer.TokenType.EOF)
                {
                    if (tokenizer.CurrentToken == ExpressionTokenizer.TokenType.Dollar)
                    {
                        tokenizer.GetNextToken();
                        if (tokenizer.CurrentToken == ExpressionTokenizer.TokenType.LeftCurlyBrace)
                        {
                            tokenizer.IgnoreWhitespace    = true;
                            tokenizer.SingleCharacterMode = false;
                            tokenizer.GetNextToken();

                            string val = Convert.ToString(eval.Evaluate(tokenizer), CultureInfo.InvariantCulture);
                            output.Append(val);
                            tokenizer.IgnoreWhitespace = false;

                            if (tokenizer.CurrentToken != ExpressionTokenizer.TokenType.RightCurlyBrace)
                            {
                                throw new ExpressionParseException("'}' expected", tokenizer.CurrentPosition.CharIndex);
                            }
                            tokenizer.SingleCharacterMode = true;
                            tokenizer.GetNextToken();
                        }
                        else
                        {
                            if (tokenizer.CurrentToken != ExpressionTokenizer.TokenType.Dollar)
                            {
                                output.Append('$');
                            }
                            if (tokenizer.CurrentToken != ExpressionTokenizer.TokenType.EOF)
                            {
                                output.Append(tokenizer.TokenText);
                                tokenizer.GetNextToken();
                            }
                        }
                    }
                    else
                    {
                        output.Append(tokenizer.TokenText);
                        tokenizer.GetNextToken();
                    }
                }
                return(output.ToString());
            }
            catch (ExpressionParseException ex)
            {
                StringBuilder errorMessage     = new StringBuilder();
                string        reformattedInput = input;

                // replace CR, LF and TAB with a space
                reformattedInput = reformattedInput.Replace('\n', ' ');
                reformattedInput = reformattedInput.Replace('\r', ' ');
                reformattedInput = reformattedInput.Replace('\t', ' ');

                errorMessage.Append(ex.Message);
                errorMessage.Append(Environment.NewLine);

                string label = "Expression: ";

                errorMessage.Append(label);
                errorMessage.Append(reformattedInput);

                int p0 = ex.StartPos;
                int p1 = ex.EndPos;

                if (p0 != -1 || p1 != -1)
                {
                    errorMessage.Append(Environment.NewLine);
                    if (p1 == -1)
                    {
                        p1 = p0 + 1;
                    }

                    for (int i = 0; i < p0 + label.Length; ++i)
                    {
                        errorMessage.Append(' ');
                    }
                    for (int i = p0; i < p1; ++i)
                    {
                        errorMessage.Append('^');
                    }
                }

                throw new BuildException(errorMessage.ToString(), location,
                                         ex.InnerException);
            }
        }
 public object Evaluate(ExpressionTokenizer tokenizer)
 {
     _evalMode = EvalMode.Evaluate;
     _tokenizer = tokenizer;
     return ParseExpression();
 }
        protected object SafeConvert(Type returnType, object source, string description, ExpressionTokenizer.Position p0, ExpressionTokenizer.Position p1)
        {
            try {
                //
                // TODO - Convert.ChangeType() is very liberal. It allows you to convert "true" to Double (1.0).
                // We shouldn't allow this. Add more cases like this here.
                //
                bool disallow = false;

                if (source == null) {
                    if (returnType == typeof(string)) {
                        return string.Empty;
                    }

                    throw BuildParseError(string.Format(CultureInfo.InvariantCulture,
                        ResourceUtils.GetString("NA1045"),
                        description, GetSimpleTypeName(returnType)), p0, p1);
                }

                if (source is bool) {
                    if (returnType != typeof(string) && returnType != typeof(bool)) {
                        // boolean can only be converted to string or boolean
                        disallow = true;
                    }
                }

                if (returnType == typeof(bool)) {
                    if (!(source is string || source is bool)) {
                        // only string and boolean can be converted to boolean
                        disallow = true;
                    }
                }

                if (source is DateTime) {
                    if (returnType != typeof(string) && returnType != typeof(DateTime)) {
                        // DateTime can only be converted to string or DateTime
                        disallow = true;
                    }
                }

                if (returnType == typeof(DateTime)) {
                    if (!(source is DateTime || source is string)) {
                        // only string and DateTime can be converted to DateTime
                        disallow = true;
                    }
                }

                if (source is TimeSpan && returnType != typeof(TimeSpan)) {
                    // implicit conversion from TimeSpan is not supported, as
                    // TimeSpan does not implement IConvertible
                    disallow = true;
                }

                if (returnType == typeof(TimeSpan) && !(source is TimeSpan)) {
                    // implicit conversion to TimeSpan is not supported
                    disallow = true;
                }

                if (returnType == typeof(string)) {
                    if (source is DirectoryInfo) {
                        return ((DirectoryInfo) source).FullName;
                    } else if (source is FileInfo) {
                        return ((FileInfo) source).FullName;
                    }
                }

                if (returnType.IsEnum) {
                    string sourceText = source as string;
                    if (sourceText != null) {
                        // support both ',' and ' ' as separator chars for flags
                        string[] flags = sourceText.Split(' ', ',');
                        StringBuilder sb = new StringBuilder(sourceText.Length);
                        for (int i = 0; i < flags.Length; i++) {
                            string flag = flags[i].Trim();
                            if (flag.Length == 0)
                                continue;
                            if (sb.Length > 0)
                                sb.Append(',');
                            sb.Append(flag);
                        }
                        return Enum.Parse(returnType, sb.ToString(), true);
                    } else {
                        return Enum.ToObject(returnType, source);
                    }
                }

                if (disallow) {
                    throw BuildParseError(string.Format(CultureInfo.InvariantCulture,
                        "Cannot convert {0} to '{1}' (actual type was '{2}').",
                        description, GetSimpleTypeName(returnType),
                        GetSimpleTypeName(source.GetType())), p0, p1);
                }

                return Convert.ChangeType(source, returnType, CultureInfo.InvariantCulture);
            } catch (ExpressionParseException) {
                throw;
            } catch (Exception ex) {
                throw BuildParseError(string.Format(CultureInfo.InvariantCulture,
                    "Cannot convert {0} to '{1}' (actual type was '{2}').",
                    description, GetSimpleTypeName(returnType),
                    GetSimpleTypeName(source.GetType())), p0, p1, ex);
            }
        }
 protected ExpressionParseException BuildParseError(string desc, ExpressionTokenizer.Position p0, ExpressionTokenizer.Position p1, Exception ex)
 {
     return new ExpressionParseException(desc, p0.CharIndex, p1.CharIndex, ex);
 }
示例#9
0
        /// <summary>
        /// Evaluates the given expression string and returns the result
        /// </summary>
        /// <param name="input"></param>
        /// <param name="location"></param>
        /// <param name="state"></param>
        /// <param name="visiting"></param>
        /// <returns></returns>
        private string EvaluateEmbeddedExpressions(string input, Location location, Hashtable state, Stack visiting)
        {
            if (input == null) {
                return null;
            }

            if (input.IndexOf('$') < 0) {
                return input;
            }

            try {
                StringBuilder output = new StringBuilder(input.Length);

                ExpressionTokenizer tokenizer = new ExpressionTokenizer();
                ExpressionEvaluator eval = new ExpressionEvaluator(Project, this, state, visiting);

                tokenizer.IgnoreWhitespace = false;
                tokenizer.SingleCharacterMode = true;
                tokenizer.InitTokenizer(input);

                while (tokenizer.CurrentToken != ExpressionTokenizer.TokenType.EOF) {
                    if (tokenizer.CurrentToken == ExpressionTokenizer.TokenType.Dollar) {
                        tokenizer.GetNextToken();
                        if (tokenizer.CurrentToken == ExpressionTokenizer.TokenType.LeftCurlyBrace) {
                            tokenizer.IgnoreWhitespace = true;
                            tokenizer.SingleCharacterMode = false;
                            tokenizer.GetNextToken();

                            string val = Convert.ToString(eval.Evaluate(tokenizer), CultureInfo.InvariantCulture);
                            output.Append(val);
                            tokenizer.IgnoreWhitespace = false;

                            if (tokenizer.CurrentToken != ExpressionTokenizer.TokenType.RightCurlyBrace) {
                                throw new ExpressionParseException("'}' expected", tokenizer.CurrentPosition.CharIndex);
                            }
                            tokenizer.SingleCharacterMode = true;
                            tokenizer.GetNextToken();
                        } else {
                            output.Append('$');
                            if (tokenizer.CurrentToken != ExpressionTokenizer.TokenType.EOF) {
                                output.Append(tokenizer.TokenText);
                                tokenizer.GetNextToken();
                            }
                        }
                    } else {
                        output.Append(tokenizer.TokenText);
                        tokenizer.GetNextToken();
                    }
                }
                return output.ToString();
            } catch (ExpressionParseException ex) {
                StringBuilder errorMessage = new StringBuilder();
                string reformattedInput = input;

                // replace CR, LF and TAB with a space
                reformattedInput = reformattedInput.Replace('\n', ' ');
                reformattedInput = reformattedInput.Replace('\r', ' ');
                reformattedInput = reformattedInput.Replace('\t', ' ');

                errorMessage.Append(ex.Message);
                errorMessage.Append(Environment.NewLine);

                string label = "Expression: ";

                errorMessage.Append(label);
                errorMessage.Append(reformattedInput);

                int p0 = ex.StartPos;
                int p1 = ex.EndPos;

                if (p0 != -1 || p1 != -1) {
                    errorMessage.Append(Environment.NewLine);
                    if (p1 == -1)
                        p1 = p0 + 1;

                    for (int i = 0; i < p0 + label.Length; ++i)
                        errorMessage.Append(' ');
                    for (int i = p0; i < p1; ++i)
                        errorMessage.Append('^');
                }

                throw new BuildException(errorMessage.ToString(), location,
                    ex.InnerException);
            }
        }
示例#10
0
        protected object SafeConvert(Type returnType, object source, string description, ExpressionTokenizer.Position p0, ExpressionTokenizer.Position p1)
        {
            try {
                //
                // TODO - Convert.ChangeType() is very liberal. It allows you to convert "true" to Double (1.0).
                // We shouldn't allow this. Add more cases like this here.
                //
                bool disallow = false;

                if (source == null) {
                    if (returnType == typeof(string)) {
                        return string.Empty;
                    }

                    throw BuildParseError(string.Format(CultureInfo.InvariantCulture,
                        ResourceUtils.GetString("NA1045"),
                        description, GetSimpleTypeName(returnType)), p0, p1);
                }

                if (source is bool) {
                    if (returnType != typeof(string) && returnType != typeof(bool)) {
                        // boolean can only be converted to string or boolean
                        disallow = true;
                    }
                }

                if (returnType == typeof(bool)) {
                    if (!(source is string || source is bool)) {
                        // only string and boolean can be converted to boolean
                        disallow = true;
                    }
                }

                if (source is DateTime) {
                    if (returnType != typeof(string) && returnType != typeof(DateTime)) {
                        // DateTime can only be converted to string or DateTime
                        disallow = true;
                    }
                }

                if (returnType == typeof(DateTime)) {
                    if (!(source is DateTime || source is string)) {
                        // only string and DateTime can be converted to DateTime
                        disallow = true;
                    }
                }

                if (source is TimeSpan && returnType != typeof(TimeSpan)) {
                    // implicit conversion from TimeSpan is not supported, as
                    // TimeSpan does not implement IConvertible
                    disallow = true;
                }

                if (returnType == typeof(TimeSpan) && !(source is TimeSpan)) {
                    // implicit conversion to TimeSpan is not supported
                    disallow = true;
                }

                if (returnType == typeof(string)) {
                    if (source is DirectoryInfo) {
                        return ((DirectoryInfo) source).FullName;
                    } else if (source is FileInfo) {
                        return ((FileInfo) source).FullName;
                    }
                }

                if (returnType.IsEnum) {
                    if (source is string) {
                        return Enum.Parse(returnType, (string) source, false);
                    } else {
                        return Enum.ToObject(returnType, source);
                    }
                }

                if (disallow) {
                    throw BuildParseError(string.Format(CultureInfo.InvariantCulture,
                        "Cannot convert {0} to '{1}' (actual type was '{2}').",
                        description, GetSimpleTypeName(returnType),
                        GetSimpleTypeName(source.GetType())), p0, p1);
                }

                return Convert.ChangeType(source, returnType, CultureInfo.InvariantCulture);
            } catch (ExpressionParseException) {
                throw;
            } catch (Exception ex) {
                throw BuildParseError(string.Format(CultureInfo.InvariantCulture,
                    "Cannot convert {0} to '{1}' (actual type was '{2}').",
                    description, GetSimpleTypeName(returnType),
                    GetSimpleTypeName(source.GetType())), p0, p1, ex);
            }
        }