public override IToken Evaluate(IToken first, IToken last, TokenTreeList parameters, bool isFinal)
        {
            if (first == null)
                throw new Exception($"Operation {Text} can not be unary.");

            IToken evaluated = last.Evaluate(parameters, isFinal);
            if (evaluated == null)
                throw new Exception($"Second element of Operation {Text} is not unique.");

            ListToken evaluatedList = evaluated as ListToken;
            if (evaluatedList != null)
            {
                ListToken list = new ListToken();
                foreach (IToken item in evaluatedList.Tokens)
                    list.Tokens.Add(Evaluate(first, item, parameters, isFinal));
                return list;
            }

            IntToken intToken = evaluated as IntToken;
            if (intToken == null)
            {
                if (isFinal)
                    throw new Exception($"Operation {Text} must have integer second element.");
                return new ExpressionToken(first, new IndexOperator(), last);
            }

            IToken tokenList = first.Evaluate(parameters, isFinal);
            ListToken listToken = tokenList as ListToken;
            int index = intToken.Value;
            return listToken == null
                ? (index == 0 && tokenList is ITypeToken ? tokenList : new ExpressionToken(first, new IndexOperator(), intToken))
                : (listToken.Tokens.Count > index ? listToken.Tokens[index] : new NullToken());
        }
 private static IToken AddToList(IToken firstList, IToken lastList)
 {
     ListToken result = new ListToken();
     ListToken tokens = firstList as ListToken;
     if (tokens != null)
         result.Tokens.AddRange(tokens.Tokens);
     else
         result.Tokens.Add(firstList);
     result.Add(lastList);
     return result;
 }
        /// <summary>
        /// Evaluate the function.
        /// </summary>
        /// <param name="parameters">The tokens that make up the function parameter list.</param>
        /// <param name="substitutions">The tokens that can be used for substitutions.</param>
        /// <param name="isFinal">Whether a result needs to be returned.</param>
        /// <returns></returns>
        public override IToken Perform(IToken parameters, TokenTreeList substitutions, bool isFinal)
        {
            ListToken listToken = parameters as ListToken;

            if (listToken == null)
                throw new Exception($"Next token must be list for '{ID}'");

            List<IToken> lastList = listToken.Tokens;
            int count = lastList.Count;
            IToken iterandKey = lastList[count - 2];
            IToken iterandIndex = null;
            ListToken iterandList = iterandKey as ListToken;
            if (iterandList != null)
            {
                if (iterandList.Tokens.Count != 2)
                    throw new Exception($"Can only have 1 or 2 iterators for '{ID}'");
                iterandIndex = iterandList.Tokens[1];
                iterandKey = iterandList.Tokens[0];
            }
            IToken method = lastList[count - 1];
            ListToken tokens = new ListToken();
            for (int i = 0; i < count - 2; i++)
            {
                IToken token = lastList[i];
                ListToken list = token as ListToken;
                if (list == null)
                {
                    list = new ListToken();
                    list.Tokens.Add(token);
                }
                int index = 0;
                foreach (IToken item in list.Tokens)
                {
                    TokenTree tree = new TokenTree();
                    tree.Children.Add(new TokenTree(iterandKey.Text, item));
                    if (iterandIndex != null)
                        tree.Children.Add(new TokenTree(iterandIndex.Text, new IntToken(index)));
                    IToken toCall = method.SubstituteParameters(tree);
                    IToken parsed = toCall.Evaluate(substitutions, isFinal);
                    if (parsed is ExpressionToken)
                    {
                        if (!isFinal)
                            return UnParsed(listToken);
                    }
                    else if (!(parsed is NullToken))
                    {
                        tokens.Add(parsed);
                    }
                    ++index;
                }
            }
            return tokens;
        }
 public override IToken Perform(IToken token, TokenTreeList parameters, bool isFinal)
 {
     ListToken list = token as ListToken;
     if (list == null)
     {
         list = new ListToken();
         list.Tokens.Add(token);
     }
     ListToken result = new ListToken();
     result.Tokens.AddRange(list.Tokens);
     result.Tokens.Reverse();
     return result;
 }
        /// <summary>
        /// Evaluate the function.
        /// </summary>
        /// <param name="parameters">The tokens that make up the function parameter list.</param>
        /// <param name="substitutions">The tokens that can be used for substitutions.</param>
        /// <param name="isFinal">Whether a result needs to be returned.</param>
        /// <returns></returns>
        public override IToken Perform(IToken parameters, TokenTreeList substitutions, bool isFinal)
        {
            ListToken listToken = parameters as ListToken;

            if (listToken == null)
                return new ListToken {new ListToken {parameters, new IntToken(1)}};

            Dictionary<string, int> found = new Dictionary<string, int>();
            foreach (IToken child in listToken.Tokens)
                if (!AddToken(found, child))
                    return UnParsed(listToken);

            ListToken list = new ListToken();
            foreach (var item in found)
                list.Add(new ListToken {new StringToken(item.Key), new IntToken(item.Value)});
            return list;
        }
        public override IToken Perform(IToken dataToken, TokenTreeList parameters, bool isFinal)
        {
            ListToken listToken = dataToken as ListToken;
            if (listToken != null)
            {
                ListToken returnList = new ListToken();
                foreach (IToken token in listToken.Tokens)
                    returnList.Add(Perform(token, parameters, isFinal));
                return returnList;
            }

            ITypeToken typeToken = dataToken as ITypeToken;
            if (typeToken != null)
                return new DoubleToken(typeToken.Convert<double>());

            throw new Exception($"Token must be list or convertible to double for {ID}");
        }
        /// <summary>
        /// Evaluate the function.
        /// </summary>
        /// <param name="parameters">The tokens that make up the function parameter list.</param>
        /// <param name="substitutions">The tokens that can be used for substitutions.</param>
        /// <param name="isFinal">Whether a result needs to be returned.</param>
        /// <returns></returns>
        public override IToken Perform(IToken parameters, TokenTreeList substitutions, bool isFinal)
        {
            string toSplit;
            int maxCount = -1;
            string[] splitOn = {" ", "\t"};
            StringSplitOptions options = StringSplitOptions.RemoveEmptyEntries;
            ListToken listToken = parameters as ListToken;

            if (listToken == null)
            {
                if (parameters is ExpressionToken)
                    return UnParsed(parameters);
                toSplit = parameters.Text;
            }
            else
            {
                List<IToken> lastList = listToken.Tokens;
                int count = lastList.Count;
                if (count != 2 && count != 3)
                    throw new Exception($"Must have 1, 2 or 3 values for '{ID}': {listToken}");

                if (lastList[0] is ExpressionToken)
                    return UnParsed(listToken);

                toSplit = lastList[0].Text;
                if (count > 1)
                {
                    string text = lastList[1].Text;
                    if (text != " ")
                    {
                        splitOn = new[] {text};
                        options = StringSplitOptions.None;
                    }
                }
                if (count > 2)
                    maxCount = lastList[2].Convert<int>();
            }

            ListToken result = new ListToken();
            string[] bits = maxCount <= 0 ? toSplit.Split(splitOn, options) : toSplit.Split(splitOn, maxCount, options);
            foreach (string bit in bits)
                result.Add(new StringToken(bit.Trim()));
            return result;
        }
 public void TestCount()
 {
     CountFunction function = new CountFunction();
     ListToken list = null;
     IToken result = function.Perform(list, null, true);
     Assert.IsInstanceOfType(result, typeof(IntToken));
     Assert.AreEqual(0, ((IntToken)result).Value);
     list = new ListToken();
     result = function.Perform(list, null, true);
     Assert.IsInstanceOfType(result, typeof(IntToken));
     Assert.AreEqual(0, ((IntToken)result).Value);
     list.Add(new BoolTooken(true));
     result = function.Perform(list, null, true);
     Assert.IsInstanceOfType(result, typeof(IntToken));
     Assert.AreEqual(1, ((IntToken)result).Value);
     list.Add(new BoolTooken(true));
     result = function.Perform(list, null, true);
     Assert.IsInstanceOfType(result, typeof(IntToken));
     Assert.AreEqual(2, ((IntToken)result).Value);
 }
        public override IToken Evaluate(IToken first, IToken last, TokenTreeList parameters, bool isFinal)
        {
            if (parameters == null)
            {
                if (isFinal)
                    throw new Exception($"Operation {Text} must have parameters if final.");
                return new ExpressionToken(first, this, last);
            }

            if (first != null)
                throw new Exception($"Operation {Text} is unary.");
            if (last == null)
                throw new Exception($"Operation {Text} needs a variable.");

            IToken evaluated = last.Evaluate(parameters, isFinal);
            if (evaluated is ExpressionToken)
                return new ExpressionToken(null, this, evaluated);
            ListToken listToken = evaluated as ListToken;
            if (listToken != null && listToken.Tokens.Exists(x => x is ExpressionToken))
                return new ExpressionToken(null, this, evaluated);

            string text = evaluated.Text;
            TokenTreeList found = parameters.FindMatches(text, true);
            ListToken result = new ListToken();
            foreach (TokenTree tokenTree in found)
            {
                bool debug = bool.Parse(tokenTree["Debug"] ?? "False");
                if (debug)
                    LogControl?.SetLogging(true);
                IToken token = tokenTree.Value.Evaluate(parameters, isFinal);
                if (!(token is NullToken))
                    result.Add(token);
                if (debug)
                    LogControl?.ResetLoggingToDefault();
            }

            if (result.Tokens.Count == 0)
                return new ExpressionToken(null, this, evaluated);
            return result.Tokens.Count == 1 ? result.Tokens[0] : result;
        }
        /// <summary>
        /// Evaluate the function.
        /// </summary>
        /// <param name="parameters">The tokens that make up the function parameter list.</param>
        /// <param name="substitutions">The tokens that can be used for substitutions.</param>
        /// <param name="isFinal">Whether a result needs to be returned.</param>
        /// <returns></returns>
        public override IToken Perform(IToken parameters, TokenTreeList substitutions, bool isFinal)
        {
            ListToken list = parameters as ListToken;
            if (list == null)
            {
                list = new ListToken();
                list.Tokens.Add(parameters);
            }

            List<IToken> parameterList = new List<IToken>();
            foreach (IToken item in list)
            {
                IToken toAdd = item;
                if (toAdd is ExpressionToken && !isFinal)
                    return UnParsed(parameters);
                parameterList.Add(toAdd);
            }

            ListToken result = new ListToken();
            switch (parameterList.Count)
            {
                case 1:
                    for (int i = 1; i <= parameterList[0].Convert<int>(); ++i)
                        result.Tokens.Add(new IntToken(i));
                    break;
                case 2:
                    for (int i = parameterList[0].Convert<int>(); i <= parameterList[1].Convert<int>(); ++i)
                        result.Tokens.Add(new IntToken(i));
                    break;
                case 3:
                    for (int i = parameterList[0].Convert<int>(); i <= parameterList[1].Convert<int>(); i += parameterList[2].Convert<int>())
                        result.Tokens.Add(new IntToken(i));
                    break;
                default:
                    throw new Exception($"Must have between 1 and 3 values for '{ID}': {parameters}");
            }
            return result;
        }
        public override IToken Evaluate(IToken first, IToken last, TokenTreeList parameters, bool isFinal)
        {
            if (_function == null)
            {
                if (first == null)
                    throw new Exception($"Operation {Text} can not be unary.");

                IToken functionToken = first.Evaluate(parameters, isFinal);
                if (functionToken == null || functionToken is ListToken)
                    throw new Exception($"First element of Operation {Text} is not unique.");

                string function = functionToken.Text;
                switch (function)
                {
                    case AndFunction.ID:
                        _function = new AndFunction();
                        break;
                    case AggregateFunction.ID:
                        _function = new AggregateFunction();
                        break;
                    case CaseFunction.ID:
                        _function = new CaseFunction();
                        break;
                    case ComparisonFunction.ID:
                        _function = new ComparisonFunction();
                        break;
                    case ContainsFunction.ID:
                        _function = new ContainsFunction();
                        break;
                    case CountFunction.ID:
                        _function = new CountFunction();
                        break;
                    case DoubleFunction.ID:
                        _function = new DoubleFunction();
                        break;
                    case IfFunction.ID:
                        _function = new IfFunction();
                        break;
                    case IntFunction.ID:
                        _function = new IntFunction();
                        break;
                    case JoinFunction.ID:
                        _function = new JoinFunction();
                        break;
                    case OrFunction.ID:
                        _function = new OrFunction();
                        break;
                    case OverFunction.ID:
                        _function = new OverFunction();
                        break;
                    case RangeFunction.ID:
                        _function = new RangeFunction();
                        break;
                    case RegexFunction.ID:
                        _function = new RegexFunction();
                        break;
                    case ReverseFunction.ID:
                        _function = new ReverseFunction();
                        break;
                    case SplitFunction.ID:
                        _function = new SplitFunction();
                        break;
                    case SumFunction.ID:
                        _function = new SumFunction();
                        break;
                    case UserFunction.ID:
                        _function = new UserFunction();
                        break;
                    default:
                        ListToken newList = new ListToken
                        {
                            new ExpressionToken(null, new SubstitutionOperator(), new StringToken(function))
                        };
                        ListToken oldList = last.Evaluate(parameters, isFinal) as ListToken;
                        if (oldList != null)
                        {
                            foreach (IToken token in oldList.Tokens)
                                newList.Add(token);
                        }
                        else
                        {
                            newList.Add(last);
                        }
                        ExpressionToken expression = new ExpressionToken(null, new FunctionOperator(new UserFunction()), newList);
                        return expression.Evaluate(parameters, isFinal);
                }
            }

            IToken parameterList;
            if (isFinal && _function is UserFunction)
            {
                parameterList = PrepareUserFunction(last, parameters);
            }
            else if (_function.IsComparisonFunction)
            {
                parameterList = last.EvaluateList();
            }
            else
            {
                parameterList = last.Evaluate(parameters, !_function.FinalCanBeExpression && isFinal);
                ExpressionToken expression = parameterList as ExpressionToken;
                if (expression != null)
                {
                    if (isFinal)
                    {
                        IToken substitute = _function.ValueIfFinalValueIsExpression(expression);
                        if (substitute is NullToken)
                            return new ExpressionToken(null, new FunctionOperator(_function), parameterList);
                        parameterList = substitute;
                    }
                    else
                    {
                        return new ExpressionToken(null, new FunctionOperator(_function), parameterList);
                    }
                }
            }

            return _function.Perform(parameterList, parameters, isFinal);
        }
        private IToken PrepareUserFunction(IToken last, TokenTreeList parameters)
        {
            ListToken list = new ListToken();
            ListToken toAdd = (ListToken)last;
            IToken toParse = toAdd.Tokens[0];
            IToken method = toParse;
            foreach (TokenTree parameter in parameters)
            {
                TokenTree onlyGlobal = parameter.Clone();

                // Make sure that parameters relevant only to higher levels do not get passed through.
                int i = 1;
                while (onlyGlobal.Children.Remove(i.ToString())) ++i;

                method = toParse.SubstituteParameters(onlyGlobal);
                if (method.Text != toParse.Text)
                    break;
            }
            list.Tokens.Add(method);
            for (int i = 1; i < toAdd.Tokens.Count; ++i)
                list.Tokens.Add(toAdd.Tokens[i].Evaluate(parameters, !_function.FinalCanBeExpression));
            return list;
        }