예제 #1
0
        /// <summary>
        /// Enumerates through the tokens, searching for tokens XX,XX,XX where XX are arguments to the function and possibly a sub expression.
        /// When a ')' is found then the end of the arguments is presumed to have been found.
        /// </summary>
        /// <param name="tokensEnum">Expected to be currently pointing to the name of the function. The next token SHOULD be a '('</param>
        private IConstruct[] GetFunctionArguments(PeekableEnumerator <Token> tokensEnum, List <Variable> currentVariables)
        {
            var arguments    = new List <IConstruct>();
            var functionName = tokensEnum.Current.Value;

            if (!(tokensEnum.MoveNext() && tokensEnum.Current.Type == TokenType.LeftParenthesis))
            {
                throw new InvalidOperationException(String.Format("{0} arguments; first token should be '(' not '{1}'", functionName, tokensEnum.Current.Value));
            }
            else if (tokensEnum.Current.Type == TokenType.LeftParenthesis && tokensEnum.CanPeek && tokensEnum.Peek.Type == TokenType.RightParenthesis)
            {
                // No arguments were specified - empty parentheses were specified
                tokensEnum.MoveNext();      // consume the left parenthesis token and point it to the right parenthesis token - i.e. the end of the function
            }
            else
            {
                bool reachedEndOfArguments = false;

                while (!reachedEndOfArguments)
                {
                    arguments.Add(GetConstructFromTokens(GetFunctionArgumentTokens(functionName, tokensEnum, currentVariables), currentVariables));

                    // tokensEnum.Current will be the last token processed by GetFunctionArgumentTokens()
                    if (tokensEnum.Current.Type == TokenType.RightParenthesis)
                    {
                        reachedEndOfArguments = true;
                    }
                }
            }

            return(arguments.ToArray());
        }
예제 #2
0
        private string[] GetCommandArguments(PeekableEnumerator <Token> tokensEnum)
        {
            var arguments    = new List <string>();
            var functionName = tokensEnum.Current.Value;

            while (tokensEnum.MoveNext())
            {
                arguments.Add(tokensEnum.Current.Value);
            }

            return(arguments.ToArray());
        }
예제 #3
0
        private IConstruct TranslateIdentifierToken(PeekableEnumerator <Token> tokensEnum, bool isInitial)
        {
            var identifierToken = tokensEnum.Current;

            var reservedWordForToken = reservedWords.SingleOrDefault(reserverWord => identifierToken == reserverWord.Word);
            var commandForToken      = Program.Commands.SingleOrDefault(aCommand => identifierToken == aCommand.Name);

            if (reservedWordForToken != null)
            {
                return(reservedWordForToken.Construct);
            }
            else if (commandForToken != null && isInitial)
            {
                return(new CommandOperation(commandForToken, GetCommandArguments(tokensEnum)));
            }
            else
            {
                if (tokensEnum.CanPeek && tokensEnum.Peek.Type == TokenType.LeftParenthesis)
                {
                    var functionForToken = Program.Functions.SingleOrDefault(aFunction => identifierToken == aFunction.Name);

                    if (functionForToken == null)
                    {
                        return(new Error(String.Format("Function '{0}' is undefined", identifierToken)));
                    }
                    else
                    {
                        return(new FunctionOperation(functionForToken, GetFunctionArguments(tokensEnum)));
                    }
                }
                else
                {
                    // ensure there is only one Variable instance for the same variable name
                    var variable = Program.Variables.SingleOrDefault(aVariable => identifierToken == aVariable.Name);

                    if (variable == null)
                    {
                        // return null;
                        var newVariable = new Variable(tokensEnum.Current.Value);
                        Program.Variables.Add(newVariable);
                        return(newVariable);
                    }
                    else
                    {
                        return(variable);
                    }
                }
            }
        }
예제 #4
0
        /// <summary>
        /// Translates an identifier as either a function, variable name or key word.
        /// A function will match a registered function name and have a left parenthesis token following it, otherwise it is a variable.
        /// </summary>
        private IConstruct TranslateIdentifierToken(PeekableEnumerator <Token> tokensEnum, List <Variable> currentVariables)
        {
            var identifierToken = tokensEnum.Current;

            var reservedWordForToken = reservedWords.SingleOrDefault(reserverWord => identifierToken == reserverWord.Word);

            if (reservedWordForToken != null)
            {
                return(reservedWordForToken.Construct);
            }
            else
            {
                if (tokensEnum.CanPeek && tokensEnum.Peek.Type == TokenType.LeftParenthesis)
                {
                    var functionForToken = allFunctions.SingleOrDefault(aFunction => identifierToken == aFunction.Name);

                    if (functionForToken == null)
                    {
                        throw new InvalidOperationException(String.Format("Function '{0}' is undefined", identifierToken));
                    }
                    else
                    {
                        return(new FunctionOperation(functionForToken, GetFunctionArguments(tokensEnum, currentVariables)));
                    }
                }
                else
                {
                    // ensure there is only one Variable instance for the same variable name
                    var variable = currentVariables.SingleOrDefault(aVariable => identifierToken == aVariable.Name);

                    if (variable == null)
                    {
                        var newVariable = new Variable(tokensEnum.Current.Value);
                        currentVariables.Add(newVariable);
                        return(newVariable);
                    }
                    else
                    {
                        return(variable);
                    }
                }
            }
        }
예제 #5
0
        /// <summary>
        /// Gets the function's next argument's tokens by traversing the tokens until the next , or ) is found (which is not within a function).
        /// Does not return the , or ) character that terminated the argument expression - it is also consumed.
        /// </summary>
        /// <param name="functionName">Only used in order to provide useful exceptions / errors.</param>
        /// <param name="tokensEnum">Should be pointing to the token that indicates the start of a function argument; either a ( or , character.</param>
        private static List <Token> GetFunctionArgumentTokens(string functionName, PeekableEnumerator <Token> tokensEnum, List <Variable> currentVariables)
        {
            var argumentTokens = new List <Token> ();

            int  functionDepth        = 0;
            bool reachedEndOfArgument = false;

            while (!reachedEndOfArgument && tokensEnum.MoveNext())
            {
                var token = tokensEnum.Current;

                // found the argument's terminating comma or right parenthesis
                if (functionDepth == 0 && (token.Type == TokenType.Comma || token.Type == TokenType.RightParenthesis))
                {
                    reachedEndOfArgument = true;
                }
                else
                {
                    argumentTokens.Add(token);

                    if (token.Type == TokenType.LeftParenthesis)
                    {
                        functionDepth++;
                    }
                    else if (token.Type == TokenType.RightParenthesis)
                    {
                        functionDepth--;
                    }
                }
            }

            if (argumentTokens.Count == 0)
            {
                throw new InvalidOperationException(String.Format("{0} has an empty argument", functionName));
            }
            else if (!reachedEndOfArgument)
            {
                throw new InvalidOperationException(String.Format("{0} is missing a terminating argument character; ',' or ')'", functionName));
            }

            return(argumentTokens);
        }
        /// <summary>
        /// Preliminary tokenization of the expression.
        /// Tokenizes numeric values, alpha values, parentheses, commas and other tokens.
        /// Any whitespace is removed.
        /// </summary>
        public static List <Token> Parse(string expression)
        {
            const char LeftParenthesis   = '(';
            const char RightParenthesis  = ')';
            const char Comma             = ',';
            const char NumericNegative   = '-';
            const char DateTimeDelimiter = '#';

            var whitespaceCharacters = new[] { ' ', '\t' };
            var numericCharacters    = new[] { '.', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' };

            // Identifier can contain . to support JSON Path
            var  identifierCharacters          = new[] { '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z' };
            var  identifierSecondaryCharacters = new[] { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '.' /* Fred*/ };    // other characters that can be used as identifiers - but cannot be a starting character
            var  textDelimiters    = new[] { '\"', '\'' };
            bool isNumericNegative = false;
            bool parsingText       = false;
            bool parsingDateTime   = false;

            var  tokens               = new List <Token>();
            var  currentTokenType     = TokenType.Other;
            var  currentToken         = String.Empty;
            char currentTextDelimiter = '\0';
            var  characterTokenType   = TokenType.Other;
            var  expressionEnumerator = new PeekableEnumerator <char>(expression);
            var  characterString      = String.Empty;

            while (expressionEnumerator.MoveNext())
            {
                var tokenIsSeparateCharacter = false;
                var character = expressionEnumerator.Current;

                // if the character is a '-' and the subsequent character is a numeric character then this is a negative number.
                // otherwise it is some other character TokenType.Other -- probably a subtraction operator.
                isNumericNegative = character == NumericNegative && expressionEnumerator.CanPeek && numericCharacters.Contains(expressionEnumerator.Peek);

                if (textDelimiters.Contains(character) || parsingText)
                {
                    if (textDelimiters.Contains(character) && !parsingText)         // started parsing
                    {
                        characterTokenType   = TokenType.Text;
                        characterString      = String.Empty;    // consume character
                        currentTextDelimiter = character;
                        parsingText          = true;
                    }
                    else if (character == currentTextDelimiter && parsingText) // finished parsing
                    {
                        characterString = String.Empty;                        // consume character
                        parsingText     = false;
                    }
                    else
                    {
                        characterString = character.ToString();
                    }
                }
                else if (character == DateTimeDelimiter || parsingDateTime)
                {
                    if (!parsingDateTime)                       // started parsing
                    {
                        characterTokenType = TokenType.DateTime;
                        characterString    = String.Empty;  // consume character
                        parsingDateTime    = true;
                    }
                    else if (character == DateTimeDelimiter) // finished parsing
                    {
                        characterString = String.Empty;      // consume character
                        parsingDateTime = false;
                    }
                    else
                    {
                        characterString = character.ToString();
                    }
                }
                else if (whitespaceCharacters.Contains(character))
                {
                    characterTokenType = TokenType.Whitespace;
                    characterString    = String.Empty;          // consume character
                }
                //else if(character == '.' && currentTokenType == TokenType.Identifier)
                //{

                //}
                else if (identifierCharacters.Contains(character) || (currentTokenType == TokenType.Identifier && identifierSecondaryCharacters.Contains(character)))
                {
                    characterTokenType = TokenType.Identifier;
                    characterString    = character.ToString();
                }
                else if (numericCharacters.Contains(character) || isNumericNegative)
                {
                    characterTokenType = TokenType.Number;
                    characterString    = character.ToString();
                }
                else if (character == LeftParenthesis)
                {
                    characterTokenType       = TokenType.LeftParenthesis;
                    characterString          = character.ToString();
                    tokenIsSeparateCharacter = true;
                }
                else if (character == RightParenthesis)
                {
                    characterTokenType       = TokenType.RightParenthesis;
                    characterString          = character.ToString();
                    tokenIsSeparateCharacter = true;
                }
                else if (character == Comma)
                {
                    characterTokenType       = TokenType.Comma;
                    characterString          = character.ToString();
                    tokenIsSeparateCharacter = true;
                }
                else
                {
                    characterTokenType = TokenType.Other;
                    characterString    = character.ToString();
                }

                if (currentTokenType == characterTokenType && !tokenIsSeparateCharacter)
                {
                    currentToken += characterString;
                }
                else
                {
                    if (currentToken.Length > 0 || currentTokenType == TokenType.Text)
                    {
                        tokens.Add(new Token(currentToken, currentTokenType));
                    }

                    currentToken     = characterString;
                    currentTokenType = characterTokenType;
                }
            }

            if (currentToken.Length > 0 || currentTokenType == TokenType.Text)
            {
                tokens.Add(new Token(currentToken, currentTokenType));
            }

            return(tokens);
        }
예제 #7
0
        private List <TranslatedToken> TranslateTokens(List <Token> tokens)
        {
            var translatedTokens = new List <TranslatedToken>();
            var tokensEnum       = new PeekableEnumerator <Token>(tokens);

            bool isInitial = true;

            while (tokensEnum.MoveNext())
            {
                var token = tokensEnum.Current;

                switch (token.Type)
                {
                case TokenType.Number:
                    translatedTokens.Add(new ConstructToken(Number.Parse(token.Value)));
                    break;

                case TokenType.Identifier:
                    var operationForTokenIdentifier = allOperators
                                                      .Select(item => item.Operation)
                                                      .SingleOrDefault(item => item.Token.Equals(token.Value));

                    if (operationForTokenIdentifier != null)
                    {
                        translatedTokens.Add(new OperatorToken(operationForTokenIdentifier));
                    }
                    else
                    {
                        translatedTokens.Add(new ConstructToken(TranslateIdentifierToken(tokensEnum, isInitial)));
                    }
                    break;

                case TokenType.LeftParenthesis:
                    translatedTokens.Add(new LeftParenthesisToken());
                    break;

                case TokenType.RightParenthesis:
                    translatedTokens.Add(new RightParenthesisToken());
                    break;

                case TokenType.Text:
                    translatedTokens.Add(new ConstructToken(new Text(token.Value)));
                    break;

                case TokenType.DateTime:
                    try
                    {
                        translatedTokens.Add(new ConstructToken(new DateTime(System.DateTime.Parse(token.Value))));
                    }
                    catch
                    {
                        translatedTokens.Add(new ConstructToken(new Text(token.Value)));
                    }
                    break;

                case TokenType.Other:
                    var operationForToken = allOperators
                                            .Select(item => item.Operation)
                                            .SingleOrDefault(item => item.Token.Equals(token.Value));

                    if (operationForToken != null)
                    {
                        translatedTokens.Add(new OperatorToken(operationForToken));
                    }
                    else
                    {
                        Output.Text(token.Value + " in an unknown operation");
                    }

                    break;

                case TokenType.Comma:
                    break;

                default:
                    throw new NotImplementedException();
                }

                isInitial = false;
            }

            if (translatedTokens.Count == 0)
            {
                Output.Text(string.Format("Token Error: {0}", tokensEnum.Current));
            }
            return(translatedTokens);
        }
예제 #8
0
        /// <summary>
        /// Translates the tokens into meaningful functions, operations and values.
        /// </summary>
        private List <TranslatedToken> TranslateTokens(List <Token> tokens, List <Variable> currentVariables)
        {
            var translatedTokens = new List <TranslatedToken>();
            var tokensEnum       = new PeekableEnumerator <Token>(tokens);

            while (tokensEnum.MoveNext())
            {
                var token = tokensEnum.Current;

                switch (token.Type)
                {
                case TokenType.Number:
                    translatedTokens.Add(new ConstructToken(Number.Parse(token.Value)));
                    break;

                case TokenType.Identifier:
                    var operationForTokenIdentifier = allOperators
                                                      .Select(item => item.Operation)
                                                      .SingleOrDefault(item => item.Token.Equals(token.Value));

                    if (operationForTokenIdentifier != null)
                    {
                        translatedTokens.Add(new OperatorToken(operationForTokenIdentifier));
                    }
                    else
                    {
                        translatedTokens.Add(new ConstructToken(TranslateIdentifierToken(tokensEnum, currentVariables)));
                    }
                    break;

                case TokenType.LeftParenthesis:
                    translatedTokens.Add(new LeftParenthesisToken());
                    break;

                case TokenType.RightParenthesis:
                    translatedTokens.Add(new RightParenthesisToken());
                    break;

                case TokenType.Text:
                    translatedTokens.Add(new ConstructToken(new Text(token.Value)));
                    break;

                case TokenType.DateTime:
                    translatedTokens.Add(new ConstructToken(new DateTime(System.DateTime.Parse(token.Value))));
                    break;

                case TokenType.Other:
                    var operationForToken = allOperators
                                            .Select(item => item.Operation)
                                            .SingleOrDefault(item => item.Token.Equals(token.Value));

                    if (operationForToken != null)
                    {
                        translatedTokens.Add(new OperatorToken(operationForToken));
                    }
                    else
                    {
                        throw new InvalidOperationException(token.Value + " in an unknown operation");
                    }

                    break;

                default:
                    throw new NotImplementedException();
                }
            }

            return(translatedTokens);
        }