internal void Append(RawToken token)
{
if (token != null)
{
ValueBuilder.Append(token.ValueBuilder);
}
}
static RawToken NewTokenIfNecessary(List <RawToken> tokens, RawToken lastToken, RawTokenType curTokenType, int position)
{
if (lastToken == null || lastToken.TokenType != curTokenType ||
curTokenType == RawTokenType.Symbol) // for symbol always let it be by itself
{
lastToken = new RawToken(curTokenType, position);
tokens.Add(lastToken);
}
return(lastToken);
}
/// <summary>
/// Splits the specified input into a list of <see cref="RawToken"/> values using white space and symbols.
/// The tokens can be recombined to rebuild the original input exactly.
/// </summary>
/// <param name="input">The input.</param>
/// <returns></returns>
public RawToken[] Tokenize(string input)
{
var tokens = new List <RawToken>();
if (input != null)
{
RawToken lastToken = null;
for (int i = 0; i < input.Length; i++)
{
var ch = input[i];
if (char.IsWhiteSpace(ch))
{
lastToken = NewTokenIfNecessary(tokens, lastToken, RawTokenType.WhiteSpace, i);
}
else if (_symbols.Contains(ch))
{
lastToken = NewTokenIfNecessary(tokens, lastToken, RawTokenType.Symbol, i);
}
else
{
lastToken = NewTokenIfNecessary(tokens, lastToken, RawTokenType.Literal, i);
}
if (ch == '\\' && ++i < input.Length)
{
// assume escape and just append next char as-is
var next = input[i];
lastToken.ValueBuilder.Append(next);
}
else
{
lastToken.ValueBuilder.Append(ch);
}
}
}
return(tokens.ToArray());
}
ExpressionToken HandleNonOperatorSymbolToken(ListReader <RawToken> reader, ExpressionToken lastExpToken, RawToken curRawToken)
{
switch (curRawToken.Value)
{
case ",":
lastExpToken = new ExpressionToken {
TokenType = ExpressionTokenType.Comma
};
_currentTokens.Add(lastExpToken);
lastExpToken.Append(curRawToken);
break;
case "(":
// if last one is string make it a function
if (lastExpToken != null && lastExpToken.TokenType == ExpressionTokenType.Value)
{
lastExpToken.TokenType = ExpressionTokenType.Function;
}
lastExpToken = new ExpressionToken {
TokenType = ExpressionTokenType.OpenParenthesis
};
_currentTokens.Add(lastExpToken);
lastExpToken.Append(curRawToken);
break;
case ")":
lastExpToken = new ExpressionToken {
TokenType = ExpressionTokenType.CloseParenthesis
};
_currentTokens.Add(lastExpToken);
lastExpToken.Append(curRawToken);
break;
case "{":
// read until end of }
lastExpToken = ReadToLiteralAs(reader, "}", ExpressionTokenType.Field);
break;
case "\"":
// read until end of "
lastExpToken = ReadToLiteralAs(reader, "\"", ExpressionTokenType.DoubleQuoted);
break;
case "'":
// read until end of '
lastExpToken = ReadToLiteralAs(reader, "'", ExpressionTokenType.SingleQuoted);
break;
}
return(lastExpToken);
}
