public IEnumerable <Token> Tokenize(string input) { if (string.IsNullOrEmpty(input)) { return(Enumerable.Empty <Token>()); } var context = new TokenizerContext(input); foreach (var c in context.FormulaChars) { Token tokenSeparator; if (CharIsTokenSeparator(c, out tokenSeparator)) { if (context.IsInString && tokenSeparator.TokenType != TokenType.String) { context.AppendToCurrentToken(c); continue; } // two operators in sequence could be "<=" or ">=" if (IsPartOfMultipleCharSeparator(context, c)) { context.AppendToLastToken(c.ToString()); context.NewToken(); continue; } if (tokenSeparator.TokenType == TokenType.String) { if (context.LastToken != null && context.LastToken.TokenType == TokenType.String && !context.CurrentTokenHasValue) { // We are dealing with an empty string (''). context.AddToken(new Token(string.Empty, TokenType.StringContent)); } context.ToggleIsInString(); } if (context.CurrentTokenHasValue) { context.AddToken(CreateToken(context)); } if (tokenSeparator.Value == "-") { if (TokenIsNegator(context)) { context.AddToken(new Token("-", TokenType.Negator)); continue; } } context.AddToken(tokenSeparator); context.NewToken(); continue; } context.AppendToCurrentToken(c); } if (context.CurrentTokenHasValue) { context.AddToken(CreateToken(context)); } return(context.Result); }
public IEnumerable<Token> Tokenize(string input) { if (string.IsNullOrEmpty(input)) { return Enumerable.Empty<Token>(); } var context = new TokenizerContext(input); foreach (var c in context.FormulaChars) { Token tokenSeparator; if(CharIsTokenSeparator(c, out tokenSeparator)) { if (context.IsInString && tokenSeparator.TokenType != TokenType.String) { context.AppendToCurrentToken(c); continue; } // two operators in sequence could be "<=" or ">=" if (IsPartOfMultipleCharSeparator(context, c)) { context.AppendToLastToken(c.ToString()); context.NewToken(); continue; } if (tokenSeparator.TokenType == TokenType.String) { if (context.LastToken != null && context.LastToken.TokenType == TokenType.String && !context.CurrentTokenHasValue) { // We are dealing with an empty string (''). context.AddToken(new Token(string.Empty, TokenType.StringContent)); } context.ToggleIsInString(); } if (context.CurrentTokenHasValue) { context.AddToken(CreateToken(context)); } if (tokenSeparator.Value == "-") { if (TokenIsNegator(context)) { context.AddToken(new Token("-", TokenType.Negator)); continue; } } context.AddToken(tokenSeparator); context.NewToken(); continue; } context.AppendToCurrentToken(c); } if (context.CurrentTokenHasValue) { context.AddToken(CreateToken(context)); } return context.Result; }
private bool IsPartOfMultipleCharSeparator(TokenizerContext context, char c) { var lastToken = context.LastToken != null ? context.LastToken.Value : string.Empty; return(_tokenProvider.IsOperator(lastToken) && _tokenProvider.IsPossibleLastPartOfMultipleCharOperator(c.ToString()) && !context.CurrentTokenHasValue); }
private Token CreateToken(TokenizerContext context) { if (context.CurrentToken == "-") { if (context.LastToken == null && context.LastToken.TokenType == TokenType.Operator) { return(new Token("-", TokenType.Negator)); } } return(_tokenFactory.Create(context.Result, context.CurrentToken)); }
private static bool TokenIsNegator(TokenizerContext context) { return(context.LastToken == null || context.LastToken.TokenType == TokenType.Operator || context.LastToken.TokenType == TokenType.OpeningBracket || context.LastToken.TokenType == TokenType.Comma || context.LastToken.TokenType == TokenType.OpeningEnumerable); }
private static bool TokenIsNegator(TokenizerContext context) { return context.LastToken == null || context.LastToken.TokenType == TokenType.Operator || context.LastToken.TokenType == TokenType.OpeningBracket || context.LastToken.TokenType == TokenType.Comma || context.LastToken.TokenType == TokenType.OpeningEnumerable; }
private bool IsPartOfMultipleCharSeparator(TokenizerContext context, char c) { var lastToken = context.LastToken != null ? context.LastToken.Value : string.Empty; return _tokenProvider.IsOperator(lastToken) && _tokenProvider.IsPossibleLastPartOfMultipleCharOperator(c.ToString()) && !context.CurrentTokenHasValue; }
private Token CreateToken(TokenizerContext context) { if (context.CurrentToken == "-") { if (context.LastToken == null && context.LastToken.TokenType == TokenType.Operator) { return new Token("-", TokenType.Negator); } } return _tokenFactory.Create(context.Result, context.CurrentToken); }