ExpressionToken HandleNonOperatorSymbolToken(ListReader <RawToken> reader, ExpressionToken lastExpToken, RawToken curRawToken) { switch (curRawToken.Value) { case ",": lastExpToken = new ExpressionToken { TokenType = ExpressionTokenType.Comma }; _currentTokens.Add(lastExpToken); lastExpToken.Append(curRawToken); break; case "(": // if last one is string make it a function if (lastExpToken != null && lastExpToken.TokenType == ExpressionTokenType.Value) { lastExpToken.TokenType = ExpressionTokenType.Function; } lastExpToken = new ExpressionToken { TokenType = ExpressionTokenType.OpenParenthesis }; _currentTokens.Add(lastExpToken); lastExpToken.Append(curRawToken); break; case ")": lastExpToken = new ExpressionToken { TokenType = ExpressionTokenType.CloseParenthesis }; _currentTokens.Add(lastExpToken); lastExpToken.Append(curRawToken); break; case "{": // read until end of } lastExpToken = ReadToLiteralAs(reader, "}", ExpressionTokenType.Field); break; case "\"": // read until end of " lastExpToken = ReadToLiteralAs(reader, "\"", ExpressionTokenType.DoubleQuoted); break; case "'": // read until end of ' lastExpToken = ReadToLiteralAs(reader, "'", ExpressionTokenType.SingleQuoted); break; } return(lastExpToken); }
private static void DetermineOperatorType(ListReader <ExpressionToken> reader, ExpressionToken tk) { tk.OperatorType = KnownOperators.TryMap(tk.Value); switch (tk.OperatorType) { case OperatorType.PreDecrement: case OperatorType.PreIncrement: // detect if it's really post ++ -- versions var prev = reader.Position > 1 ? reader.Peek(-2) : null; if (prev != null && prev.TokenType == ExpressionTokenType.Value) { if (tk.OperatorType == OperatorType.PreIncrement) { tk.OperatorType = OperatorType.PostIncrement; } else { tk.OperatorType = OperatorType.PostDecrement; } } break; case OperatorType.Addition: case OperatorType.Subtraction: // detect if unary + - prev = reader.Position > 1 ? reader.Peek(-2) : null; if (prev == null || (prev.TokenType == ExpressionTokenType.Operator && prev.OperatorType != OperatorType.PostDecrement && prev.OperatorType != OperatorType.PostIncrement)) { if (tk.OperatorType == OperatorType.Addition) { tk.OperatorType = OperatorType.UnaryPlus; } else { tk.OperatorType = OperatorType.UnaryMinus; } } break; case OperatorType.None: throw new NotSupportedException(string.Format(CultureInfo.InvariantCulture, "Operator {0} is not supported.", tk.Value)); } }
static void MassageTokens(List <ExpressionToken> tokens) { // do final token parsing based on contexts and cleanup var reader = new ListReader <ExpressionToken>(tokens); while (!reader.IsEnd) { var tk = reader.Read(); if (tk.TokenType == ExpressionTokenType.Operator) { // special detection for operators depending on where it is :( DetermineOperatorType(reader, tk); } tk.Freeze(); } }
ExpressionToken ReadToLiteralAs(ListReader <RawToken> reader, string literalValue, ExpressionTokenType tokenType) { ExpressionToken lastExpToken = new ExpressionToken { TokenType = tokenType }; _currentTokens.Add(lastExpToken); while (!reader.IsEnd) { var next = reader.Read(); if (next.TokenType == RawTokenType.Symbol && next.Value == literalValue) { break; } lastExpToken.Append(next); } return(lastExpToken); }
/// <summary> /// Splits the specified input into a list of <see cref="ExpressionToken" /> values. /// </summary> /// <param name="input">The input.</param> /// <returns></returns> /// <exception cref="System.NotSupportedException"></exception> public ExpressionToken[] Tokenize(string input) { _currentTokens = new List <ExpressionToken>(); ExpressionToken lastExpToken = null; var reader = new ListReader <RawToken>(new RawTokenizer().Tokenize(input)); while (!reader.IsEnd) { var curRawToken = reader.Read(); switch (curRawToken.TokenType) { case RawTokenType.WhiteSpace: // generially ends previous token outside other special scopes lastExpToken = null; break; case RawTokenType.Literal: if (lastExpToken == null || lastExpToken.TokenType != ExpressionTokenType.Value) { lastExpToken = new ExpressionToken { TokenType = ExpressionTokenType.Value }; _currentTokens.Add(lastExpToken); } lastExpToken.Append(curRawToken); break; case RawTokenType.Symbol: // first do operator match by checking the prev op // and see if combined with current token would still match a known operator if (KnownOperators.IsKnown(curRawToken.Value)) { if (lastExpToken != null && lastExpToken.TokenType == ExpressionTokenType.Operator) { var testOpValue = lastExpToken.Value + curRawToken.Value; if (KnownOperators.IsKnown(testOpValue)) { // just append it lastExpToken.Append(curRawToken); continue; } } // start new one lastExpToken = new ExpressionToken { TokenType = ExpressionTokenType.Operator }; _currentTokens.Add(lastExpToken); lastExpToken.Append(curRawToken); } else { lastExpToken = HandleNonOperatorSymbolToken(reader, lastExpToken, curRawToken); } break; default: // should never happen throw new NotSupportedException(string.Format(CultureInfo.InvariantCulture, "Unsupported token type {0} at position {1}.", curRawToken.TokenType, curRawToken.Position)); } } MassageTokens(_currentTokens); return(_currentTokens.ToArray()); }