ExpressionToken HandleNonOperatorSymbolToken(ListReader <RawToken> reader, ExpressionToken lastExpToken, RawToken curRawToken) { switch (curRawToken.Value) { case ",": lastExpToken = new ExpressionToken { TokenType = ExpressionTokenType.Comma }; _currentTokens.Add(lastExpToken); lastExpToken.Append(curRawToken); break; case "(": // if last one is string make it a function if (lastExpToken != null && lastExpToken.TokenType == ExpressionTokenType.Value) { lastExpToken.TokenType = ExpressionTokenType.Function; } lastExpToken = new ExpressionToken { TokenType = ExpressionTokenType.OpenParenthesis }; _currentTokens.Add(lastExpToken); lastExpToken.Append(curRawToken); break; case ")": lastExpToken = new ExpressionToken { TokenType = ExpressionTokenType.CloseParenthesis }; _currentTokens.Add(lastExpToken); lastExpToken.Append(curRawToken); break; case "{": // read until end of } lastExpToken = ReadToLiteralAs(reader, "}", ExpressionTokenType.Field); break; case "\"": // read until end of " lastExpToken = ReadToLiteralAs(reader, "\"", ExpressionTokenType.DoubleQuoted); break; case "'": // read until end of ' lastExpToken = ReadToLiteralAs(reader, "'", ExpressionTokenType.SingleQuoted); break; } return(lastExpToken); }
ExpressionToken ReadToLiteralAs(ListReader <RawToken> reader, string literalValue, ExpressionTokenType tokenType) { ExpressionToken lastExpToken = new ExpressionToken { TokenType = tokenType }; _currentTokens.Add(lastExpToken); while (!reader.IsEnd) { var next = reader.Read(); if (next.TokenType == RawTokenType.Symbol && next.Value == literalValue) { break; } lastExpToken.Append(next); } return(lastExpToken); }
/// <summary> /// Splits the specified input into a list of <see cref="ExpressionToken" /> values. /// </summary> /// <param name="input">The input.</param> /// <returns></returns> /// <exception cref="System.NotSupportedException"></exception> public ExpressionToken[] Tokenize(string input) { _currentTokens = new List <ExpressionToken>(); ExpressionToken lastExpToken = null; var reader = new ListReader <RawToken>(new RawTokenizer().Tokenize(input)); while (!reader.IsEnd) { var curRawToken = reader.Read(); switch (curRawToken.TokenType) { case RawTokenType.WhiteSpace: // generially ends previous token outside other special scopes lastExpToken = null; break; case RawTokenType.Literal: if (lastExpToken == null || lastExpToken.TokenType != ExpressionTokenType.Value) { lastExpToken = new ExpressionToken { TokenType = ExpressionTokenType.Value }; _currentTokens.Add(lastExpToken); } lastExpToken.Append(curRawToken); break; case RawTokenType.Symbol: // first do operator match by checking the prev op // and see if combined with current token would still match a known operator if (KnownOperators.IsKnown(curRawToken.Value)) { if (lastExpToken != null && lastExpToken.TokenType == ExpressionTokenType.Operator) { var testOpValue = lastExpToken.Value + curRawToken.Value; if (KnownOperators.IsKnown(testOpValue)) { // just append it lastExpToken.Append(curRawToken); continue; } } // start new one lastExpToken = new ExpressionToken { TokenType = ExpressionTokenType.Operator }; _currentTokens.Add(lastExpToken); lastExpToken.Append(curRawToken); } else { lastExpToken = HandleNonOperatorSymbolToken(reader, lastExpToken, curRawToken); } break; default: // should never happen throw new NotSupportedException(string.Format(CultureInfo.InvariantCulture, "Unsupported token type {0} at position {1}.", curRawToken.TokenType, curRawToken.Position)); } } MassageTokens(_currentTokens); return(_currentTokens.ToArray()); }