Пример #1
0
        private String parse(string[] tokens, Stack<string> vStk, Stack<char> sStk)
        {
            for (int i = 0; i < tokens.Length; i++)
            {
                string token = tokens[i];

                switch (token)
                {
                    case "+":
                    case "-":
                    case "*":
                    case "/":
                        while (!sStk.isEmpty())
                        {
                            if (!sStk.Peek().Equals("("))
                                if (precedence(sStk.Peek()) >= precedence(token[0]))
                                {
                                    vStk.Push(sStk.Pop() + "");
                                }
                                else
                                    break;
                        }
                        sStk.Push(token[0]);
                        break;
                    case "(":
                        sStk.Push(token[0]);
                        break;
                    case ")":
                        while (!sStk.isEmpty())
                        {
                            if (sStk.Peek().Equals('('))
                            {
                                sStk.Pop();
                                break;
                            }
                            vStk.Push(sStk.Pop() + "");
                        }
                        break;
                    default:
                        vStk.Push(token);
                        break;

                }
            }
            while (!sStk.isEmpty())
                vStk.Push(sStk.Pop() + "");

            String result = "";
            while (!vStk.isEmpty())
                result = vStk.Pop() + " " + result;
            return result.Trim();
        }
		public void convert() {

			Stack<char> stack = new Stack<char> ();

			for (int i = 0; i < infixString.Length; i++) {
				string tmp = "";
				char c = infixString [i];

				while ("0123456789.".IndexOf (c) >= 0) {
					tmp += c;
					c = infixString [++i];
				}

				if (tmp != "") {
					postfixString += tmp + " ";
					i--;
				} else if ("+-/x".IndexOf(c) >= 0) {
					// stack is empty, push operator
					if (stack.Count == 0) {
						stack.Push (c);
					} else {
						// if top of stack has higher precedence
						char topStack = stack.Peek();
						Debug.WriteLine ("Top of stack: " + topStack);
						while (hasPrecedence(topStack, c) && stack.Count > 0) {
							Debug.WriteLine ("Popping from stack because " + topStack + " has precedence over " + c);
							char popped = stack.Pop ();
							postfixString += popped + " ";
							if (stack.Count > 0)
								topStack = stack.Peek ();
	
							Debug.WriteLine ("Post-fix string: " + postfixString);
						}

						stack.Push (c);

					}
				}
			}

			foreach (char c in stack) {
				postfixString += c + " ";
			}

		}
Пример #3
0
        public string[] ConvertToPostfixNotation(string input)
        {
            List<string> operators = new List<string>(standart_operators);
            List<string> outputSeparated = new List<string>();
            Stack<string> stack = new Stack<string>();
            foreach (string c in Separate(input))
            {
                if (operators.Contains(c))
                {
                    if (stack.Count > 0 && !c.Equals("("))
                    {
                        if (c.Equals(")"))
                        {
                            string s = stack.Pop();
                            while (!s.Equals("("))
                            {
                                outputSeparated.Add(s);
                                s = stack.Pop();
                            }
                        }
                        else if (GetPriority(c) > GetPriority(stack.Peek()))
                            stack.Push(c);
                        else
                        {
                            while (stack.Count > 0 && GetPriority(c) <= GetPriority(stack.Peek()))
                                outputSeparated.Add(stack.Pop());
                            stack.Push(c);
                        }
                    }
                    else
                        stack.Push(c);
                }
                else
                    outputSeparated.Add(c);
            }
            if (stack.Count > 0)
                foreach (string c in stack)
                    outputSeparated.Add(c);

            return outputSeparated.ToArray();
        }
Пример #4
0
        private static string toPostfix(string infix)
        {
            string expr = infix;
            Stack<char> stack = new Stack<char>();
            string postfix = "";

            foreach (char chr in expr.ToArray())
            {
                if (chr == '(')
                {
                    stack.Push(chr);
                }
                else if ("+-*/".IndexOf(chr) != -1)
                {
                    while(stack.Count != 0 && priority(stack.Peek()) >= priority(chr))
                    {
                        postfix = postfix + stack.Pop();
                    }
                    stack.Push(chr);
                }
                else if (chr == ')')
                {
                    while (stack.Peek() != '(')
                    {
                        postfix = postfix + stack.Pop();
                    }
                    stack.Pop();
                }
                else
                {
                    postfix = postfix + chr;
                }
            }

            while (stack.Count != 0)
            {
                postfix = postfix + stack.Pop();
            }

            return postfix;
        }
Пример #5
0
        /// <summary>
        /// Evaluate expression with Dijkstra's Shunting Yard Algorithm
        /// </summary>
        /// <returns>result of calculations</returns>
        public double? Calculate()
        {
            if (_input == null)
                throw new ArgumentException();

            Stack<Op> operatorStack = new Stack<Op>();
            Queue<Op> outputQueue = new Queue<Op>();

            // Let's split the input string into a token list
            List<String> tokenList = Regex.Split(_input, TokenSplitRegex).Select(t => t.Trim()).Where(t => !String.IsNullOrEmpty(t)).ToList();
            for (int tokenNum = 0; tokenNum < tokenList.Count(); ++tokenNum)
            {
                double? tmpValue;
                String token = tokenList[tokenNum];
                TokenType tokenType = GetTokenType(token, out tmpValue);

                // Handle this token and insert into the correct queue or stack
                switch (tokenType)
                {
                    case TokenType.Value:
                        if (tmpValue.HasValue)
                            outputQueue.Enqueue(new Operand(tmpValue.Value));
                        else
                            throw new ArgumentException("Unknown operand " + token);
                        break;
                    case TokenType.Operator:
                        Operator newOperator = GetOperator(token, (tokenNum == 0 || tokenList[tokenNum - 1] == "("));
                        if (operatorStack.Count > 0)
                        {
                            Op topOperator = operatorStack.Peek();
                            if (topOperator is Operator)
                            {
                                if (newOperator.Precedence <= ((Operator)topOperator).Precedence)
                                {
                                    outputQueue.Enqueue(operatorStack.Pop());
                                }
                            }
                        }
                        operatorStack.Push(newOperator);
                        break;
                    case TokenType.LeftParenthesis:
                        operatorStack.Push(new LeftParenthesis());
                        break;
                    case TokenType.RightParenthesis:
                        // Handle all operators in the stack (i.e. move them to the outputQueue)
                        // until we find the LeftParenthesis
                        while (!(operatorStack.Peek() is LeftParenthesis))
                        {
                            outputQueue.Enqueue(operatorStack.Pop());
                        }
                        operatorStack.Pop();
                        break;
                    case TokenType.Function:
                        if ((tokenList.Count >= tokenNum + 1) && (tokenList[tokenNum + 1] == "("))
                        {
                            Function.FunctionTypes type = Function.GetFunctionType(token);
                            if (type == Function.FunctionTypes.UNKNOWN)
                            {
                                throw new ArgumentException("Unknown function " + token);
                            }
                            operatorStack.Push(new Function(type));
                        }
                        break;
                }

                // If we don't find any token between a value and parenthesis, automatically
                // add a multiply sign
                if (tokenType == TokenType.Value || tokenType == TokenType.RightParenthesis)
                {
                    if (tokenNum < tokenList.Count() - 1)
                    {
                        String nextToken = tokenList[tokenNum + 1];
                        TokenType nextTokenType = GetTokenType(nextToken, out tmpValue);
                        if (nextTokenType != TokenType.Operator && nextTokenType != TokenType.RightParenthesis)
                        {
                            tokenList.Insert(tokenNum + 1, "*");
                        }
                    }
                }
            }

            // Move all operators into the outputqueue
            while (operatorStack.Count > 0)
            {
                Op operand = operatorStack.Pop();
                if (operand is LeftParenthesis || operand is RightParenthesis)
                {
                    throw new ArgumentException("Mismatched parentheses");
                }

                outputQueue.Enqueue(operand);
            }

            // Now we have the expression in reverse polish notation and it's easy to calculate
            // Step through the outputQueue and calculate the result
            Stack<Operand> outputStack = new Stack<Operand>();
            while (outputQueue.Count > 0)
            {
                Op currentOp = outputQueue.Dequeue();

                if (currentOp is Operand)
                {
                    outputStack.Push((Operand)currentOp);
                }
                else if (currentOp is Operator)
                {
                    Operator currentOperator = (Operator)currentOp;
                    currentOperator.Execute(outputStack, this.Mode);
                }
            }

            // If we haven't got only one answer, the formula is invalid, return that.
            if (outputStack.Count != 1)
            {
                throw new ArgumentException("Invalid formula");
            }

            // Pop and return the result
            return outputStack.Pop().Value;
        }
Пример #6
0
        public static Element Evaluate(string s, ref int i)
        {
            int start = i;
            Stack<Element> element = new Stack<Element>();
            Stack<Operator> op = new Stack<Operator>();
            op.Push(new OpenBracket());

            for (; i < s.Length;)
            {
                char c = s[i];
                if (c == ' ')
                {
                    i++;
                    continue;
                }
                if (StrNumber.IndexOf(c) != -1)
                {
                    element.Push(getDouble(s, ref i));
                    continue;
                }

                if (StrOperator.IndexOf(c) != -1)
                {
                    if(c == '-' && (i == 0 || (StrOperator + '(').IndexOf(s[i-1]) != -1))
                    {
                        i++;
                        Negative n = new Negative();
                        n.Inputs.Add(Evaluate(s, ref i));
                        element.Push(n);
                        continue;
                    }
                    if (c == '+' && (i == 0 || (StrOperator + '(').IndexOf(s[i - 1]) != -1))
                    {
                        i++;
                        continue;
                    }
                    Operator temp = getOperator(s, ref i);
                    while (temp.Level <= op.Peek().Level && !(temp is OpenBracket))
                    {
                        Operator o = op.Pop();
                        o.MakeInput(element);
                        element.Push(o);
                    }
                    op.Push(temp);
                    continue;
                }
                if (c == '(')
                {
                    i++;
                    element.Push(Evaluate(s, ref i));
                    continue;
                }
                if (StrEnd.IndexOf(c) != -1)
                {
                    while (!(op.Peek() is OpenBracket))
                    {
                        Operator o = op.Pop();
                        o.MakeInput(element);
                        element.Push(o);
                    }
                    op.Pop();
                    break;
                }

                element.Push(getSpec(s, ref i));
            }

            if (element.Count != 1) throw new Exception("Error when calculate \"" + s.Substring(start, i - start) + "\"");

            return element.Pop();
        }
Пример #7
0
        /// <summary>
        /// Создание обратной польской записи для выражения
        /// </summary>
        /// <param name="expression">Математическое выражение</param>
        /// <returns>Список токенов в виде обратной польской записи</returns>
        public static List <Token> GetRPN(string expression)
        {
            MatchCollection collection = Regex.Matches(expression, @"\(|\)|[A-F]|¬|∨|∧|⨁|_|↓|→|↔");

            Regex variables  = new Regex(@"[A-F]");
            Regex operations = new Regex(@"¬|∨|∧|→|↔|⨁|_|↓");
            Regex brackets   = new Regex(@"\(|\)");

            string[] priority = { "¬", "∧", "∨", "⨁", "_", "↓", "→", "↔" };

            Stack <string> stack = new Stack <string>();
            List <Token>   list  = new List <Token>();

            foreach (Match match in collection)
            {
                Match temp = variables.Match(match.Value);
                if (temp.Success)
                {
                    list.Add(new Token(temp.Value, TokenType.Variable)); continue;
                }

                temp = brackets.Match(match.Value);
                if (temp.Success)
                {
                    if (temp.Value == "(")
                    {
                        stack.Push(temp.Value); continue;
                    }
                    string operation = stack.Pop();
                    while (operation != "(")
                    {
                        list.Add(new Token(operation, TokenType.Operation));
                        operation = stack.Pop();
                    }
                    continue;
                }

                temp = operations.Match(match.Value);
                if (temp.Success)
                {
                    if (stack.Count != 0)
                    {
                        while (Array.IndexOf(priority, temp.Value) > Array.IndexOf(priority, stack.Peek()))
                        {
                            if (stack.Peek() == "(")
                            {
                                break;
                            }
                            list.Add(new Token(stack.Pop(), TokenType.Operation));
                            if (stack.Count == 0)
                            {
                                break;
                            }
                        }
                    }
                    stack.Push(temp.Value);
                }
            }
            while (stack.Count != 0)
            {
                list.Add(new Token(stack.Pop(), TokenType.Operation));
            }

            return(list);
        }
        private void Equals_Click(object sender, RoutedEventArgs e)
        {
            char[] token = Operand.Text.ToCharArray();
            //Console.WriteLine(Operand.Text);
            for (int i = 0; i < token.Length; i++)
            {
                // Console.WriteLine("Token: "+token[i].ToString());
                if (token[i] == ' ')
                {
                    continue;
                }

                else if (token[i] == '(')
                {
                    signs.Push(token[i]);
                    // Console.WriteLine("Pushed left parent to signs");
                }

                else if (token[i] >= '0' && token[i] <= '9')
                {
                    //Console.WriteLine(token[i]);

                    String val = "";
                    while (i < token.Length && (token[i] >= '0' && token[i] <= '9') || i < token.Length && token[i] == '.')
                    {
                        val += (token[i++]);
                    }
                    values.Push(double.Parse(val.ToString()));
                }

                else if (token[i] == ')')
                {
                    // Console.WriteLine("In Right parent ");
                    //Console.WriteLine("the peek is" + signs.Peek().ToString());
                    while (signs.Peek() != '(')
                    {
                        double x = Calculation(values.Pop(), signs.Pop(), values.Pop());
                        values.Push(x);
                        // Console.WriteLine("Rigt parent val" + x.ToString());
                        if (signs.Count == 0)
                        {
                            display.Text = "ERROR! Unmatched parenthesis";
                            Clear();
                        }
                    }
                    signs.Pop();
                }

                else if (token[i] == '+' || token[i] == '-' || token[i] == 'x' || token[i] == '/')
                {
                    // Console.WriteLine("Debugging ");

                    while (signs.Count > 0 && (opereationPrecedence(token[i]) >= opereationPrecedence(signs.Peek())))
                    {
                        double y = Calculation(values.Pop(), signs.Pop(), values.Pop());
                        values.Push(y);
                        //Console.WriteLine("Else val " + y.ToString());
                    }
                    signs.Push(token[i]);
                }
            }

            while (signs.Count > 0)
            {
                double z = Calculation(values.Pop(), signs.Pop(), values.Pop());
                values.Push(z);
                // Console.WriteLine("Result val"+z.ToString());
            }

            display.Text = values.Pop().ToString();
            result       = true;
        }
Пример #9
0
        private static string ConvertExpressionToRPN(string inputExpression)
        {
            string output = string.Empty;
            Stack<string> operatorStack = new Stack<string>();

            int countChars;
            int legthInputExpression = inputExpression.Length;

            for (countChars = 0; countChars < legthInputExpression; countChars++)
            {
                // Parse number in input expression
                if (Char.IsDigit(inputExpression[countChars]) || inputExpression[countChars] == '.')
                {
                    while(!IsSimpleOperator(inputExpression[countChars]))
                    {
                        output += inputExpression[countChars];
                        countChars++;

                        if (countChars == legthInputExpression)
                        {
                            break;
                        }
                    }

                    output += " ";
                    countChars--;
                }

                if (IsSimpleOperator(inputExpression[countChars]))
                {
                    if (inputExpression[countChars] == '(')
                    {
                        operatorStack.Push(inputExpression[countChars].ToString());
                    }

                    if (inputExpression[countChars] == ')')
                    {
                        string itemsAtBracket = operatorStack.Pop();

                        // items from stack to output string
                        while (itemsAtBracket != "(")
                        {
                            output += itemsAtBracket + " ";
                            itemsAtBracket = operatorStack.Pop();
                        }
                    }

                    if (inputExpression[countChars] != '(' && inputExpression[countChars] != ')')
                    {
                        // Adding operator to stack if not open|close quote
                        if (operatorStack.Count() > 0 &&
                            GetPriorytyForItem(inputExpression[countChars].ToString()) <= GetPriorytyForItem(operatorStack.Peek()))
                        {
                            output += operatorStack.Pop() + " ";
                        }

                        operatorStack.Push(inputExpression[countChars].ToString());
                    }
                }
            }

            // Output items From stack to output string
            while (operatorStack.Count > 0)
            {
                output += operatorStack.Pop() + " ";
            }

            return output.Trim();
        }
Пример #10
0
        public Expression Parse(string statement)
        {
            if (string.IsNullOrEmpty(statement))
                throw new ArgumentNullException(nameof(statement));

            var position = 0;
            var operatorStack = new Stack<ArithmeticToken>();
            var outputQueue = new Queue<ArithmeticToken>();

            while (true)
            {
                var currentToken = ArithmeticToken.GetNextToken(statement, position); // position could be passed by ref, but for sake of single responsibility...

                if (currentToken.Kind == ArithmeticTokenKind.NotSupported)
                {
                    throw new InvalidArithmeticTokenException(currentToken);
                }

                if (currentToken.Kind == ArithmeticTokenKind.End) break;

                // while we have to copy position every time 
                position += currentToken.Value.Length;

                if (currentToken.Kind == ArithmeticTokenKind.Nope) continue;

                if (currentToken.Kind == ArithmeticTokenKind.Double ||
                    currentToken.Kind == ArithmeticTokenKind.Integer)
                {
                    outputQueue.Enqueue(currentToken);
                    continue;
                }

                if (currentToken.Kind == ArithmeticTokenKind.Operator)
                {
                    // there is an operator at the top of the operator stack
                    if (operatorStack.Count > 0)
                    {
                        var op = operatorStack.Peek();

                        // with greater than or equal to precedence 
                        // and the operator is left associative
                        var topStackOperatorInfo = op.GetOperatorInfo();
                        var currentOperatorInfo = currentToken.GetOperatorInfo();

                        if (topStackOperatorInfo != null &&
                            topStackOperatorInfo.IsLeft &&
                            topStackOperatorInfo.Precedence >= currentOperatorInfo.Precedence)

                            // pop operators from the operator stack, onto the output queue.
                            outputQueue.Enqueue(operatorStack.Pop());
                    }

                    // push the read operator onto the operator stack
                    operatorStack.Push(currentToken);

                    continue;
                }

                if (currentToken.Kind == ArithmeticTokenKind.Priority)
                {
                    const string LeftBracket = "(";

                    // if the token is a left bracket (i.e. "(")
                    if (currentToken.Value == LeftBracket)
                    {
                        // push it onto the operator stack.
                        operatorStack.Push(currentToken);
                        continue;
                    }

                    // if the token is a right bracket (i.e. ")")
                    // while the operator at the top of the operator stack is not a left bracket:
                    while (true)
                    {
                        // if the stack runs out without finding a left bracket, then there are
                        // mismatched parentheses
                        if (operatorStack.Count == 0)
                        {
                            throw new InvalidArithmeticStatementException("parentheses mismatched");
                        }

                        // pop operator
                        var op = operatorStack.Pop();

                        if (op.Value == LeftBracket)
                        {
                            break;
                        }

                        outputQueue.Enqueue(op);
                    }
                }
            }

            // pop the operator onto the output queue
            while (operatorStack.Count > 0)
            {
                var op = operatorStack.Pop();

                // if the operator token on the top of the stack is a bracket, then
                if (op.Kind == ArithmeticTokenKind.Priority)
                {
                    // there are mismatched parentheses
                    throw new InvalidArithmeticStatementException("parentheses mismatched");
                }

                outputQueue.Enqueue(op);
            }

            return ExpressionConverter.FromPostfixNotation(outputQueue);
        }
        /// <summary>
        /// Converts a standard infix expression to list of tokens in
        /// postfix order.
        /// </summary>
        /// <param name="expression">Expression to evaluate</param>
        /// <returns></returns>
        protected List <string> TokenizeExpression(string expression)
        {
            List <string>  tokens     = new List <string>();
            Stack <string> stack      = new Stack <string>();
            State          state      = State.None;
            int            parenCount = 0;
            string         temp;

            TextParser parser = new TextParser(expression);

            while (!parser.EndOfText)
            {
                if (Char.IsWhiteSpace(parser.Peek()))
                {
                    // Ignore spaces, tabs, etc.
                }
                else if (parser.Peek() == '(')
                {
                    // Cannot follow operand
                    if (state == State.Operand)
                    {
                        throw new EvalException(ErrOperatorExpected, parser.Position);
                    }
                    // Allow additional unary operators after "("
                    if (state == State.UnaryOperator)
                    {
                        state = State.Operator;
                    }
                    // Push opening parenthesis onto stack
                    stack.Push(parser.Peek().ToString());
                    // Track number of parentheses
                    parenCount++;
                }
                else if (parser.Peek() == ')')
                {
                    // Must follow operand
                    if (state != State.Operand)
                    {
                        throw new EvalException(ErrOperandExpected, parser.Position);
                    }
                    // Must have matching open parenthesis
                    if (parenCount == 0)
                    {
                        throw new EvalException(ErrUnmatchedClosingParen, parser.Position);
                    }
                    // Pop all operators until matching "(" found
                    temp = stack.Pop();
                    while (temp != "(")
                    {
                        tokens.Add(temp);
                        temp = stack.Pop();
                    }
                    // Track number of parentheses
                    parenCount--;
                }
                else if ("+-*/^&|⊕<>%÷×CP".Contains(parser.Peek().ToString()))
                {
                    // Need a bit of extra code to support unary operators
                    if (state == State.Operand)
                    {
                        // Pop operators with precedence >= current operator
                        int currPrecedence = GetPrecedence(parser.Peek().ToString());
                        while (stack.Count > 0 && GetPrecedence(stack.Peek()) >= currPrecedence)
                        {
                            tokens.Add(stack.Pop());
                        }
                        stack.Push(parser.Peek().ToString());
                        state = State.Operator;
                    }
                    else if (state == State.UnaryOperator)
                    {
                        // Don't allow two unary operators together
                        throw new EvalException(ErrOperandExpected, parser.Position);
                    }
                    else
                    {
                        // Test for unary operator
                        if (parser.Peek() == '-')
                        {
                            // Push unary minus
                            stack.Push(UnaryMinus);
                            state = State.UnaryOperator;
                        }
                        else if (parser.Peek() == '+')
                        {
                            // Just ignore unary plus
                            state = State.UnaryOperator;
                        }
                        else
                        {
                            throw new EvalException(ErrOperandExpected, parser.Position);
                        }
                    }
                }
                else if (Char.IsDigit(parser.Peek()) || parser.Peek() == '.')
                {
                    if (state == State.Operand)
                    {
                        // Cannot follow other operand
                        throw new EvalException(ErrOperatorExpected, parser.Position);
                    }
                    // Parse number
                    temp = ParseNumberToken(parser);
                    tokens.Add(temp);
                    state = State.Operand;
                    continue;
                }
                else
                {
                    double result;

                    // Parse symbols and functions
                    if (state == State.Operand)
                    {
                        // Symbol or function cannot follow other operand
                        throw new EvalException(ErrOperatorExpected, parser.Position);
                    }

                    // TBD ADD ADDITIONAL OPERATORS HERE
                    if (!Char.IsLetter(parser.Peek()) &&
                        parser.Peek() != '√' &&
                        parser.Peek() != '!' ||
                        parser.Peek() == '_')
                    {
                        // Invalid character
                        temp = String.Format(ErrUnexpectedCharacter, parser.Peek());
                        throw new EvalException(temp, parser.Position);
                    }
                    // Save start of symbol for error reporting
                    int symbolPos = parser.Position;
                    // Parse this symbol
                    temp = ParseSymbolToken(parser);
                    // Skip whitespace
                    parser.MovePastWhitespace();


                    // Check for parameter list
                    bool   wasString    = false;
                    string stringResult = string.Empty;
                    if (parser.Peek() == '(')
                    {
                        wasString = true;

                        // Found parameter list, evaluate function
                        stringResult = EvaluateFunction(parser, temp, symbolPos);
                        result       = Convert.ToDouble(stringResult);
                    }
                    else
                    {
                        // No parameter list, evaluate symbol (variable)
                        result = EvaluateSymbol(temp, symbolPos);
                    }
                    //// Handle negative result
                    if (result < 0)
                    {
                        stack.Push(UnaryMinus);
                        result       = Math.Abs(result);
                        stringResult = stringResult.Substring(1);
                    }

                    if (wasString)
                    {
                        tokens.Add(stringResult);
                    }
                    else
                    {
                        tokens.Add(result.ToString());
                    }

                    state = State.Operand;
                    continue;
                }
                parser.MoveAhead();
            }
            // Expression cannot end with operator
            if (state == State.Operator || state == State.UnaryOperator)
            {
                throw new EvalException(ErrOperandExpected, parser.Position);
            }
            // Check for balanced parentheses
            if (parenCount > 0)
            {
                throw new EvalException(ErrClosingParenExpected, parser.Position);
            }
            // Retrieve remaining operators from stack
            while (stack.Count > 0)
            {
                tokens.Add(stack.Pop());
            }
            return(tokens);
        }