private static void DetermineOperatorType(ListReader <ExpressionToken> reader, ExpressionToken tk) { tk.OperatorType = KnownOperators.TryMap(tk.Value); switch (tk.OperatorType) { case OperatorType.PreDecrement: case OperatorType.PreIncrement: // detect if it's really post ++ -- versions var prev = reader.Position > 1 ? reader.Peek(-2) : null; if (prev != null && prev.TokenType == ExpressionTokenType.Value) { if (tk.OperatorType == OperatorType.PreIncrement) { tk.OperatorType = OperatorType.PostIncrement; } else { tk.OperatorType = OperatorType.PostDecrement; } } break; case OperatorType.Addition: case OperatorType.Subtraction: // detect if unary + - prev = reader.Position > 1 ? reader.Peek(-2) : null; if (prev == null || (prev.TokenType == ExpressionTokenType.Operator && prev.OperatorType != OperatorType.PostDecrement && prev.OperatorType != OperatorType.PostIncrement)) { if (tk.OperatorType == OperatorType.Addition) { tk.OperatorType = OperatorType.UnaryPlus; } else { tk.OperatorType = OperatorType.UnaryMinus; } } break; case OperatorType.None: throw new NotSupportedException(string.Format(CultureInfo.InvariantCulture, "Operator {0} is not supported.", tk.Value)); } }
private void HandleOperatorToken(ExpressionToken inToken) { while (_stack.Count > 0) { var op2 = _stack.Peek(); if (op2.TokenType == ExpressionTokenType.Operator) { var op1Prec = KnownOperators.GetPrecedence(inToken.OperatorType); var op2Prec = KnownOperators.GetPrecedence(op2.OperatorType); var op1IsLeft = KnownOperators.IsLeftAssociative(inToken.OperatorType); if ((op1IsLeft && op1Prec <= op2Prec) || (!op1IsLeft && op1Prec < op2Prec)) { _output.Add(_stack.Pop()); continue; } } break; } _stack.Push(inToken); }
/// <summary> /// Splits the specified input into a list of <see cref="ExpressionToken" /> values. /// </summary> /// <param name="input">The input.</param> /// <returns></returns> /// <exception cref="System.NotSupportedException"></exception> public ExpressionToken[] Tokenize(string input) { _currentTokens = new List <ExpressionToken>(); ExpressionToken lastExpToken = null; var reader = new ListReader <RawToken>(new RawTokenizer().Tokenize(input)); while (!reader.IsEnd) { var curRawToken = reader.Read(); switch (curRawToken.TokenType) { case RawTokenType.WhiteSpace: // generially ends previous token outside other special scopes lastExpToken = null; break; case RawTokenType.Literal: if (lastExpToken == null || lastExpToken.TokenType != ExpressionTokenType.Value) { lastExpToken = new ExpressionToken { TokenType = ExpressionTokenType.Value }; _currentTokens.Add(lastExpToken); } lastExpToken.Append(curRawToken); break; case RawTokenType.Symbol: // first do operator match by checking the prev op // and see if combined with current token would still match a known operator if (KnownOperators.IsKnown(curRawToken.Value)) { if (lastExpToken != null && lastExpToken.TokenType == ExpressionTokenType.Operator) { var testOpValue = lastExpToken.Value + curRawToken.Value; if (KnownOperators.IsKnown(testOpValue)) { // just append it lastExpToken.Append(curRawToken); continue; } } // start new one lastExpToken = new ExpressionToken { TokenType = ExpressionTokenType.Operator }; _currentTokens.Add(lastExpToken); lastExpToken.Append(curRawToken); } else { lastExpToken = HandleNonOperatorSymbolToken(reader, lastExpToken, curRawToken); } break; default: // should never happen throw new NotSupportedException(string.Format(CultureInfo.InvariantCulture, "Unsupported token type {0} at position {1}.", curRawToken.TokenType, curRawToken.Position)); } } MassageTokens(_currentTokens); return(_currentTokens.ToArray()); }