public IEnumerable <Token> Tokenize(string input)
{
if (string.IsNullOrEmpty(input))
{
return(Enumerable.Empty <Token>());
}
var context = new TokenizerContext(input);
foreach (var c in context.FormulaChars)
{
Token tokenSeparator;
if (CharIsTokenSeparator(c, out tokenSeparator))
{
if (context.IsInString && tokenSeparator.TokenType != TokenType.String)
{
context.AppendToCurrentToken(c);
continue;
}
// two operators in sequence could be "<=" or ">="
if (IsPartOfMultipleCharSeparator(context, c))
{
context.AppendToLastToken(c.ToString());
context.NewToken();
continue;
}
if (tokenSeparator.TokenType == TokenType.String)
{
if (context.LastToken != null &&
context.LastToken.TokenType == TokenType.String &&
!context.CurrentTokenHasValue)
{
// We are dealing with an empty string ('').
context.AddToken(new Token(string.Empty, TokenType.StringContent));
}
context.ToggleIsInString();
}
if (context.CurrentTokenHasValue)
{
context.AddToken(CreateToken(context));
}
if (tokenSeparator.Value == "-")
{
if (TokenIsNegator(context))
{
context.AddToken(new Token("-", TokenType.Negator));
continue;
}
}
context.AddToken(tokenSeparator);
context.NewToken();
continue;
}
context.AppendToCurrentToken(c);
}
if (context.CurrentTokenHasValue)
{
context.AddToken(CreateToken(context));
}
return(context.Result);
}
public IEnumerable<Token> Tokenize(string input)
{
if (string.IsNullOrEmpty(input))
{
return Enumerable.Empty<Token>();
}
var context = new TokenizerContext(input);
foreach (var c in context.FormulaChars)
{
Token tokenSeparator;
if(CharIsTokenSeparator(c, out tokenSeparator))
{
if (context.IsInString && tokenSeparator.TokenType != TokenType.String)
{
context.AppendToCurrentToken(c);
continue;
}
// two operators in sequence could be "<=" or ">="
if (IsPartOfMultipleCharSeparator(context, c))
{
context.AppendToLastToken(c.ToString());
context.NewToken();
continue;
}
if (tokenSeparator.TokenType == TokenType.String)
{
if (context.LastToken != null &&
context.LastToken.TokenType == TokenType.String &&
!context.CurrentTokenHasValue)
{
// We are dealing with an empty string ('').
context.AddToken(new Token(string.Empty, TokenType.StringContent));
}
context.ToggleIsInString();
}
if (context.CurrentTokenHasValue)
{
context.AddToken(CreateToken(context));
}
if (tokenSeparator.Value == "-")
{
if (TokenIsNegator(context))
{
context.AddToken(new Token("-", TokenType.Negator));
continue;
}
}
context.AddToken(tokenSeparator);
context.NewToken();
continue;
}
context.AppendToCurrentToken(c);
}
if (context.CurrentTokenHasValue)
{
context.AddToken(CreateToken(context));
}
return context.Result;
}