static void MassageTokens(List <ExpressionToken> tokens)
{
// do final token parsing based on contexts and cleanup
var reader = new ListReader <ExpressionToken>(tokens);
while (!reader.IsEnd)
{
var tk = reader.Read();
if (tk.TokenType == ExpressionTokenType.Operator)
{
// special detection for operators depending on where it is :(
DetermineOperatorType(reader, tk);
}
tk.Freeze();
}
}
ExpressionToken ReadToLiteralAs(ListReader <RawToken> reader, string literalValue, ExpressionTokenType tokenType)
{
ExpressionToken lastExpToken = new ExpressionToken {
TokenType = tokenType
};
_currentTokens.Add(lastExpToken);
while (!reader.IsEnd)
{
var next = reader.Read();
if (next.TokenType == RawTokenType.Symbol && next.Value == literalValue)
{
break;
}
lastExpToken.Append(next);
}
return(lastExpToken);
}
/// <summary>
/// Splits the specified input into a list of <see cref="ExpressionToken" /> values.
/// </summary>
/// <param name="input">The input.</param>
/// <returns></returns>
/// <exception cref="System.NotSupportedException"></exception>
public ExpressionToken[] Tokenize(string input)
{
_currentTokens = new List <ExpressionToken>();
ExpressionToken lastExpToken = null;
var reader = new ListReader <RawToken>(new RawTokenizer().Tokenize(input));
while (!reader.IsEnd)
{
var curRawToken = reader.Read();
switch (curRawToken.TokenType)
{
case RawTokenType.WhiteSpace:
// generially ends previous token outside other special scopes
lastExpToken = null;
break;
case RawTokenType.Literal:
if (lastExpToken == null || lastExpToken.TokenType != ExpressionTokenType.Value)
{
lastExpToken = new ExpressionToken {
TokenType = ExpressionTokenType.Value
};
_currentTokens.Add(lastExpToken);
}
lastExpToken.Append(curRawToken);
break;
case RawTokenType.Symbol:
// first do operator match by checking the prev op
// and see if combined with current token would still match a known operator
if (KnownOperators.IsKnown(curRawToken.Value))
{
if (lastExpToken != null && lastExpToken.TokenType == ExpressionTokenType.Operator)
{
var testOpValue = lastExpToken.Value + curRawToken.Value;
if (KnownOperators.IsKnown(testOpValue))
{
// just append it
lastExpToken.Append(curRawToken);
continue;
}
}
// start new one
lastExpToken = new ExpressionToken {
TokenType = ExpressionTokenType.Operator
};
_currentTokens.Add(lastExpToken);
lastExpToken.Append(curRawToken);
}
else
{
lastExpToken = HandleNonOperatorSymbolToken(reader, lastExpToken, curRawToken);
}
break;
default:
// should never happen
throw new NotSupportedException(string.Format(CultureInfo.InvariantCulture, "Unsupported token type {0} at position {1}.", curRawToken.TokenType, curRawToken.Position));
}
}
MassageTokens(_currentTokens);
return(_currentTokens.ToArray());
}
