public TokenHandler(TokenizerContext context, ITokenFactory tokenFactory, ITokenSeparatorProvider tokenProvider, TokenSeparatorHandler tokenSeparatorHandler) { _context = context; _tokenFactory = tokenFactory; _tokenProvider = tokenProvider; _tokenSeparatorHandler = tokenSeparatorHandler; }
public IEnumerable <Token> Tokenize(string input, string worksheet) { if (string.IsNullOrEmpty(input)) { return(Enumerable.Empty <Token>()); } // MA 1401: Ignore leading plus in formula. input = input.TrimStart('+'); var context = new TokenizerContext(input); var handler = new TokenHandler(context, _tokenFactory, _separatorProvider); handler.Worksheet = worksheet; while (handler.HasMore()) { handler.Next(); } if (context.CurrentTokenHasValue) { context.AddToken(CreateToken(context, worksheet)); } CleanupTokens(context, _separatorProvider.Tokens); return(context.Result); }
private bool IsPartOfMultipleCharSeparator(TokenizerContext context, char c) { var lastToken = context.LastToken != null ? context.LastToken.Value : string.Empty; return(_tokenProvider.IsOperator(lastToken) && _tokenProvider.IsPossibleLastPartOfMultipleCharOperator(c.ToString(CultureInfo.InvariantCulture)) && !context.CurrentTokenHasValue); }
private Token CreateToken(TokenizerContext context, string worksheet) { if (context.CurrentToken == "-") { if (context.LastToken == null && context.LastToken.TokenType == TokenType.Operator) { return(new Token("-", TokenType.Negator)); } } return(_tokenFactory.Create(context.Result, context.CurrentToken, worksheet)); }
private static void HandleUnrecognizedTokens(TokenizerContext context, IDictionary <string, Token> tokens) { int i = 0; while (i < context.Result.Count) { var token = context.Result[i]; if (token.TokenType == TokenType.Unrecognized) { //Check 3-D sheet reference, Note: 3-D reference with single sheet name will be seen as a single sheet name if (i < context.Result.Count - 4 && context.Result[i + 1].TokenType == TokenType.Colon && context.Result[i + 2].TokenType == TokenType.Unrecognized && context.Result[i + 3].TokenType == TokenType.ExclamationMark && _sheetReferenceTokens.Contains(context.Result[i + 4].TokenType)) { token.TokenType = TokenType.WorksheetName; context.Result[i + 2].TokenType = TokenType.WorksheetName; i += 4; continue; } //Check regular sheet reference if (i < context.Result.Count - 2 && context.Result[i + 1].TokenType == TokenType.ExclamationMark && _sheetReferenceTokens.Contains(context.Result[i + 2].TokenType)) { token.TokenType = TokenType.WorksheetName; i += 2; continue; } //Check for function if (i < context.Result.Count - 1 && context.Result[i + 1].TokenType == TokenType.OpeningParenthesis) { token.TokenType = TokenType.Function; i += 2; continue; } //Check for Column / Row reference if (i < context.Result.Count - 2 && context.Result[i + 1].TokenType == TokenType.Colon && context.Result[i + 2].TokenType == TokenType.Unrecognized && IsColumnOrRowReference($"{token.Value}:{context.Result[i + 2].Value}")) { token.TokenType = TokenType.ExcelAddress; context.Result[i + 2].TokenType = TokenType.ExcelAddress; i += 3; continue; } token.TokenType = TokenType.NameValue; } i++; } }
private static void SetNegatorOperator(TokenizerContext context, int i, IDictionary <string, Token> tokens) { if (context.Result[i].Value == "-" && i > 0 && (context.Result[i].TokenType == TokenType.Operator || context.Result[i].TokenType == TokenType.Negator)) { if (TokenIsNegator(context.Result[i - 1])) { context.Result[i] = new Token("-", TokenType.Negator); } else { context.Result[i] = tokens["-"]; } } }
private static void SetNegatorOperator(TokenizerContext context, int i) { if (context.Result[i].Value == "-" && i > 0 && (context.Result[i].TokenType == TokenType.Operator || context.Result[i].TokenType == TokenType.Negator)) { if (TokenIsNegator(context.Result[i - 1])) { context.Result[i].TokenType = TokenType.Negator; } else { context.Result[i].TokenType = TokenType.Operator; } } }
public IEnumerable <Token> Tokenize(string input, string worksheet) { if (string.IsNullOrEmpty(input)) { return(Enumerable.Empty <Token>()); } // MA 1401: Ignore leading plus in formula. input = input.TrimStart('+'); var context = new TokenizerContext(input, worksheet, _tokenFactory); var handler = context.CreateHandler(); while (handler.HasMore()) { handler.Next(); } context.PostProcess(); return(context.Result); }
private static bool TokenIsNegator(string token, TokenizerContext context) { if (token != "-") { return(false); } if (!context.LastToken.HasValue) { return(true); } var t = context.LastToken.Value; return(t.TokenType == TokenType.Operator || t.TokenType == TokenType.OpeningParenthesis || t.TokenType == TokenType.Comma || t.TokenType == TokenType.SemiColon || t.TokenType == TokenType.OpeningEnumerable); }
public IEnumerable <Token> Tokenize(string input, string worksheet) { if (string.IsNullOrEmpty(input)) { return(Enumerable.Empty <Token>()); } var context = new TokenizerContext(input); var handler = new TokenHandler(context, _tokenFactory, _separatorProvider); handler.Worksheet = worksheet; while (handler.HasMore()) { handler.Next(); } if (context.CurrentTokenHasValue) { context.AddToken(CreateToken(context, worksheet)); } HandleUnrecognizedTokens(context, _separatorProvider.Tokens); return(context.Result); }
public IEnumerable<Token> Tokenize(string input, string worksheet) { if (string.IsNullOrEmpty(input)) { return Enumerable.Empty<Token>(); } // MA 1401: Ignore leading plus in formula. input = input.TrimStart('+'); var context = new TokenizerContext(input); var handler = new TokenHandler(context, _tokenFactory, _separatorProvider); handler.Worksheet = worksheet; while(handler.HasMore()) { handler.Next(); } if (context.CurrentTokenHasValue) { context.AddToken(CreateToken(context, worksheet)); } CleanupTokens(context, _separatorProvider.Tokens); return context.Result; }
private Token CreateToken(TokenizerContext context, string worksheet) { if (context.CurrentToken == "-") { if (context.LastToken == null && context.LastToken.TokenType == TokenType.Operator) { return new Token("-", TokenType.Negator); } } return _tokenFactory.Create(context.Result, context.CurrentToken, worksheet); }
private static bool TokenIsNegator(TokenizerContext context) { return TokenIsNegator(context.LastToken); }
private static void CleanupTokens(TokenizerContext context) { for (int i = 0; i < context.Result.Count; i++) { var token=context.Result[i]; if (token.TokenType == TokenType.Unrecognized) { if (i < context.Result.Count - 1) { if (context.Result[i+1].TokenType == TokenType.OpeningParenthesis) { token.TokenType = TokenType.Function; } else { token.TokenType = TokenType.NameValue; } } else { token.TokenType = TokenType.NameValue; } } else if ((token.TokenType == TokenType.Operator || token.TokenType == TokenType.Negator) && i < context.Result.Count - 1 && (token.Value=="+" || token.Value=="-")) { if (i > 0 && token.Value == "+") //Remove any + with an opening parenthesis before. { if (context.Result[i - 1].TokenType == TokenType.OpeningParenthesis) { context.Result.RemoveAt(i); SetNegatorOperator(context, i); i--; continue; } } var nextToken = context.Result[i + 1]; if (nextToken.TokenType == TokenType.Operator || nextToken.TokenType == TokenType.Negator) { if (token.Value == "+" && (nextToken.Value=="+" || nextToken.Value == "-")) { //Remove first context.Result.RemoveAt(i); SetNegatorOperator(context, i); i--; } else if (token.Value == "-" && nextToken.Value == "+") { //Remove second context.Result.RemoveAt(i+1); SetNegatorOperator(context, i); i--; } else if (token.Value == "-" && nextToken.Value == "-") { //Remove first and set operator to + context.Result.RemoveAt(i); if (i == 0) { context.Result.RemoveAt(i+1); i += 2; } else { context.Result[i].TokenType = TokenType.Operator; context.Result[i].Value = "+"; SetNegatorOperator(context, i); i--; } } } } } }
public TokenHandler(TokenizerContext context, ITokenFactory tokenFactory, ITokenSeparatorProvider tokenProvider) : this(context, tokenFactory, tokenProvider, new TokenSeparatorHandler(tokenProvider)) { }
private static bool TokenIsNegator(TokenizerContext context) { return(TokenIsNegator(context.LastToken)); }
public IEnumerable<Token> Tokenize(string input, string worksheet) { if (string.IsNullOrEmpty(input)) { return Enumerable.Empty<Token>(); } // MA 1401: Ignore leading plus in formula. input = input.TrimStart('+'); var context = new TokenizerContext(input); for (int i = 0; i<context.FormulaChars.Length;i++) { var c = context.FormulaChars[i]; Token tokenSeparator; if (CharIsTokenSeparator(c, out tokenSeparator)) { if (context.IsInString) { if (tokenSeparator.TokenType == TokenType.String && i + 1 < context.FormulaChars.Length && context.FormulaChars[i + 1] == '\'') { i++; context.AppendToCurrentToken(c); continue; } else if(tokenSeparator.TokenType != TokenType.String) { context.AppendToCurrentToken(c); continue; } } if (tokenSeparator.TokenType == TokenType.OpeningBracket) { context.AppendToCurrentToken(c); context.BracketCount++; continue; } if (tokenSeparator.TokenType == TokenType.ClosingBracket) { context.AppendToCurrentToken(c); context.BracketCount--; continue; } if (context.BracketCount > 0) { context.AppendToCurrentToken(c); continue; } // two operators in sequence could be "<=" or ">=" if (IsPartOfMultipleCharSeparator(context, c)) { var sOp = context.LastToken.Value + c.ToString(CultureInfo.InvariantCulture); var op = _tokenProvider.Tokens[sOp]; context.ReplaceLastToken(op); context.NewToken(); continue; } if (tokenSeparator.TokenType == TokenType.String) { if (context.LastToken != null && context.LastToken.TokenType == TokenType.OpeningEnumerable) { context.AppendToCurrentToken(c); context.ToggleIsInString(); continue; } else if (context.LastToken != null && context.LastToken.TokenType == TokenType.String && !context.CurrentTokenHasValue) //Added check for enumartion { // We are dealing with an empty string (''). context.AddToken(new Token(string.Empty, TokenType.StringContent)); } context.ToggleIsInString(); } if (context.CurrentTokenHasValue) { context.AddToken(CreateToken(context, worksheet)); //If the a next token is an opening parantheses and the previous token is interpeted as an address or name, then the currenct token is a function if(tokenSeparator.TokenType==TokenType.OpeningParenthesis && (context.LastToken.TokenType==TokenType.ExcelAddress || context.LastToken.TokenType==TokenType.NameValue)) { context.LastToken.TokenType=TokenType.Function; } } if (tokenSeparator.Value == "-") { if (TokenIsNegator(context)) { context.AddToken(new Token("-", TokenType.Negator)); continue; } } context.AddToken(tokenSeparator); context.NewToken(); continue; } context.AppendToCurrentToken(c); } if (context.CurrentTokenHasValue) { context.AddToken(CreateToken(context, worksheet)); } CleanupTokens(context); return context.Result; }
private static bool IsDoubleQuote(Token tokenSeparator, int formulaCharIndex, TokenizerContext context) { return(tokenSeparator.TokenType == TokenType.String && formulaCharIndex + 1 < context.FormulaChars.Length && context.FormulaChars[formulaCharIndex + 1] == '\"'); }
private static void CleanupTokens(TokenizerContext context, IDictionary <string, Token> tokens) { for (int i = 0; i < context.Result.Count; i++) { var token = context.Result[i]; if (token.TokenType == TokenType.Unrecognized) { if (i < context.Result.Count - 1) { if (context.Result[i + 1].TokenType == TokenType.OpeningParenthesis) { token.TokenType = TokenType.Function; } else { token.TokenType = TokenType.NameValue; } } else { token.TokenType = TokenType.NameValue; } } else if (token.TokenType == TokenType.WorksheetName) { // use this and the following three tokens token.TokenType = TokenType.ExcelAddress; var sb = new StringBuilder(); var nToRemove = 3; if (context.Result.Count < i + nToRemove) { token.TokenType = TokenType.InvalidReference; nToRemove = context.Result.Count - i - 1; } else if (context.Result[i + 3].TokenType != TokenType.ExcelAddress) { token.TokenType = TokenType.InvalidReference; nToRemove--; } else { for (var ix = 0; ix < 4; ix++) { sb.Append(context.Result[i + ix].Value); } } token.Value = sb.ToString(); for (var ix = 0; ix < nToRemove; ix++) { context.Result.RemoveAt(i + 1); } } else if ((token.TokenType == TokenType.Operator || token.TokenType == TokenType.Negator) && i < context.Result.Count - 1 && (token.Value == "+" || token.Value == "-")) { if (i > 0 && token.Value == "+") //Remove any + with an opening parenthesis before. { if (context.Result[i - 1].TokenType == TokenType.OpeningParenthesis) { context.Result.RemoveAt(i); SetNegatorOperator(context, i, tokens); i--; continue; } } var nextToken = context.Result[i + 1]; if (nextToken.TokenType == TokenType.Operator || nextToken.TokenType == TokenType.Negator) { if (token.Value == "+" && (nextToken.Value == "+" || nextToken.Value == "-")) { //Remove first context.Result.RemoveAt(i); SetNegatorOperator(context, i, tokens); i--; } else if (token.Value == "-" && nextToken.Value == "+") { //Remove second context.Result.RemoveAt(i + 1); SetNegatorOperator(context, i, tokens); i--; } else if (token.Value == "-" && nextToken.Value == "-") { //Remove first and set operator to + context.Result.RemoveAt(i); if (i == 0) { context.Result.RemoveAt(i + 1); i += 2; } else { //context.Result[i].TokenType = TokenType.Operator; //context.Result[i].Value = "+"; context.Result[i] = tokens["+"]; SetNegatorOperator(context, i, tokens); i--; } } } } } }
public void Init() { _tokenizerContext = new TokenizerContext("test"); InitHandler(_tokenizerContext); }
private static void CleanupTokens(TokenizerContext context) { for (int i = 0; i < context.Result.Count; i++) { var token = context.Result[i]; if (token.TokenType == TokenType.Unrecognized) { if (i < context.Result.Count - 1) { if (context.Result[i + 1].TokenType == TokenType.OpeningParenthesis) { token.TokenType = TokenType.Function; } else { token.TokenType = TokenType.NameValue; } } else { token.TokenType = TokenType.NameValue; } } else if ((token.TokenType == TokenType.Operator || token.TokenType == TokenType.Negator) && i < context.Result.Count - 1 && (token.Value == "+" || token.Value == "-")) { if (i > 0 && token.Value == "+") //Remove any + with an opening parenthesis before. { if (context.Result[i - 1].TokenType == TokenType.OpeningParenthesis) { context.Result.RemoveAt(i); SetNegatorOperator(context, i); i--; continue; } } var nextToken = context.Result[i + 1]; if (nextToken.TokenType == TokenType.Operator || nextToken.TokenType == TokenType.Negator) { if (token.Value == "+" && (nextToken.Value == "+" || nextToken.Value == "-")) { //Remove first context.Result.RemoveAt(i); SetNegatorOperator(context, i); i--; } else if (token.Value == "-" && nextToken.Value == "+") { //Remove second context.Result.RemoveAt(i + 1); SetNegatorOperator(context, i); i--; } else if (token.Value == "-" && nextToken.Value == "-") { //Remove first and set operator to + context.Result.RemoveAt(i); if (i == 0) { context.Result.RemoveAt(i + 1); i += 2; } else { context.Result[i].TokenType = TokenType.Operator; context.Result[i].Value = "+"; SetNegatorOperator(context, i); i--; } } } } } }
private void FixOperators(TokenizerContext context) { }
public TokenHandler(TokenizerContext context, ITokenFactory tokenFactory, ITokenSeparatorProvider tokenProvider) { _context = context; _tokenFactory = tokenFactory; _tokenProvider = tokenProvider; }
private bool IsPartOfMultipleCharSeparator(TokenizerContext context, char c) { var lastToken = context.LastToken != null ? context.LastToken.Value : string.Empty; return _tokenProvider.IsOperator(lastToken) && _tokenProvider.IsPossibleLastPartOfMultipleCharOperator(c.ToString(CultureInfo.InvariantCulture)) && !context.CurrentTokenHasValue; }
private static void SetNegatorOperator(TokenizerContext context, int i, IDictionary<string, Token> tokens) { if (context.Result[i].Value == "-" && i > 0 && (context.Result[i].TokenType == TokenType.Operator || context.Result[i].TokenType == TokenType.Negator)) { if (TokenIsNegator(context.Result[i - 1])) { context.Result[i] = new Token("-", TokenType.Negator); } else { context.Result[i] = tokens["-"]; } } }
private static void CleanupTokens(TokenizerContext context, IDictionary<string, Token> tokens) { for (int i = 0; i < context.Result.Count; i++) { var token=context.Result[i]; if (token.TokenType == TokenType.Unrecognized) { if (i < context.Result.Count - 1) { if (context.Result[i+1].TokenType == TokenType.OpeningParenthesis) { token.TokenType = TokenType.Function; } else { token.TokenType = TokenType.NameValue; } } else { token.TokenType = TokenType.NameValue; } } else if(token.TokenType == TokenType.WorksheetName){ // use this and the following three tokens token.TokenType = TokenType.ExcelAddress; var sb = new StringBuilder(); var nToRemove = 3; if (context.Result.Count < i + nToRemove) { token.TokenType = TokenType.InvalidReference; nToRemove = context.Result.Count - i - 1; } else if(context.Result[i + 3].TokenType != TokenType.ExcelAddress) { token.TokenType = TokenType.InvalidReference; nToRemove--; } else { for (var ix = 0; ix < 4; ix++) { sb.Append(context.Result[i + ix].Value); } } token.Value = sb.ToString(); for(var ix = 0; ix < nToRemove; ix++) { context.Result.RemoveAt(i + 1); } } // Clean up leading '+' along with the following operator combinations: ++, --, +-, -+ else if ((token.TokenType == TokenType.Operator || token.TokenType == TokenType.Negator) && i < context.Result.Count - 1 && (token.Value=="+" || token.Value=="-")) { //Remove '+' from start of formula and formula arguments if (token.Value == "+" && (i == 0 || context.Result[i - 1].TokenType == TokenType.OpeningParenthesis || context.Result[i - 1].TokenType == TokenType.Comma)) { context.Result.RemoveAt(i); SetNegatorOperator(context, i, tokens); i--; continue; } var nextToken = context.Result[i + 1]; if (nextToken.TokenType == TokenType.Operator || nextToken.TokenType == TokenType.Negator) { // Remove leading '+' from operator combinations if (token.Value == "+" && (nextToken.Value=="+" || nextToken.Value == "-")) { context.Result.RemoveAt(i); SetNegatorOperator(context, i, tokens); i--; } // Remove trailing '+' from a negator operation else if (token.Value == "-" && nextToken.Value == "+") { context.Result.RemoveAt(i+1); SetNegatorOperator(context, i, tokens); i--; } // Convert double negator operation to positive declaration else if (token.Value == "-" && nextToken.Value == "-") { context.Result.RemoveAt(i); context.Result[i] = tokens["+"]; i--; } } } } }
private static void CleanupTokens(TokenizerContext context, IDictionary<string, Token> tokens) { for (int i = 0; i < context.Result.Count; i++) { var token=context.Result[i]; if (token.TokenType == TokenType.Unrecognized) { if (i < context.Result.Count - 1) { if (context.Result[i+1].TokenType == TokenType.OpeningParenthesis) { token.TokenType = TokenType.Function; } else { token.TokenType = TokenType.NameValue; } } else { token.TokenType = TokenType.NameValue; } } else if(token.TokenType == TokenType.WorksheetName){ // use this and the following three tokens token.TokenType = TokenType.ExcelAddress; var sb = new StringBuilder(); var nToRemove = 3; if (context.Result.Count < i + nToRemove) { token.TokenType = TokenType.InvalidReference; nToRemove = context.Result.Count - i - 1; } else if(context.Result[i + 3].TokenType != TokenType.ExcelAddress) { token.TokenType = TokenType.InvalidReference; nToRemove--; } else { for (var ix = 0; ix < 4; ix++) { sb.Append(context.Result[i + ix].Value); } } token.Value = sb.ToString(); for(var ix = 0; ix < nToRemove; ix++) { context.Result.RemoveAt(i + 1); } } else if ((token.TokenType == TokenType.Operator || token.TokenType == TokenType.Negator) && i < context.Result.Count - 1 && (token.Value=="+" || token.Value=="-")) { if (i > 0 && token.Value == "+") //Remove any + with an opening parenthesis before. { if (context.Result[i - 1].TokenType == TokenType.OpeningParenthesis) { context.Result.RemoveAt(i); SetNegatorOperator(context, i, tokens); i--; continue; } } var nextToken = context.Result[i + 1]; if (nextToken.TokenType == TokenType.Operator || nextToken.TokenType == TokenType.Negator) { if (token.Value == "+" && (nextToken.Value=="+" || nextToken.Value == "-")) { //Remove first context.Result.RemoveAt(i); SetNegatorOperator(context, i, tokens); i--; } else if (token.Value == "-" && nextToken.Value == "+") { //Remove second context.Result.RemoveAt(i+1); SetNegatorOperator(context, i, tokens); i--; } else if (token.Value == "-" && nextToken.Value == "-") { //Remove first and set operator to + context.Result.RemoveAt(i); if (i == 0) { context.Result.RemoveAt(i+1); i += 2; } else { //context.Result[i].TokenType = TokenType.Operator; //context.Result[i].Value = "+"; context.Result[i] = tokens["+"]; SetNegatorOperator(context, i, tokens); i--; } } } } } }
public IEnumerable <Token> Tokenize(string input, string worksheet) { if (string.IsNullOrEmpty(input)) { return(Enumerable.Empty <Token>()); } // MA 1401: Ignore leading plus in formula. input = input.TrimStart('+'); var context = new TokenizerContext(input); for (int i = 0; i < context.FormulaChars.Length; i++) { var c = context.FormulaChars[i]; Token tokenSeparator; if (CharIsTokenSeparator(c, out tokenSeparator)) { if (context.IsInString) { if (tokenSeparator.TokenType == TokenType.String && i + 1 < context.FormulaChars.Length && context.FormulaChars[i + 1] == '\'') { i++; context.AppendToCurrentToken(c); continue; } else if (tokenSeparator.TokenType != TokenType.String) { context.AppendToCurrentToken(c); continue; } } if (tokenSeparator.TokenType == TokenType.OpeningBracket) { context.AppendToCurrentToken(c); context.BracketCount++; continue; } if (tokenSeparator.TokenType == TokenType.ClosingBracket) { context.AppendToCurrentToken(c); context.BracketCount--; continue; } if (context.BracketCount > 0) { context.AppendToCurrentToken(c); continue; } // two operators in sequence could be "<=" or ">=" if (IsPartOfMultipleCharSeparator(context, c)) { var sOp = context.LastToken.Value + c.ToString(CultureInfo.InvariantCulture); var op = _tokenProvider.Tokens[sOp]; context.ReplaceLastToken(op); context.NewToken(); continue; } if (tokenSeparator.TokenType == TokenType.String) { if (context.LastToken != null && context.LastToken.TokenType == TokenType.OpeningEnumerable) { context.AppendToCurrentToken(c); context.ToggleIsInString(); continue; } else if (context.LastToken != null && context.LastToken.TokenType == TokenType.String && !context.CurrentTokenHasValue) //Added check for enumartion { // We are dealing with an empty string (''). context.AddToken(new Token(string.Empty, TokenType.StringContent)); } context.ToggleIsInString(); } if (context.CurrentTokenHasValue) { context.AddToken(CreateToken(context, worksheet)); //If the a next token is an opening parantheses and the previous token is interpeted as an address or name, then the currenct token is a function if (tokenSeparator.TokenType == TokenType.OpeningParenthesis && (context.LastToken.TokenType == TokenType.ExcelAddress || context.LastToken.TokenType == TokenType.NameValue)) { context.LastToken.TokenType = TokenType.Function; } } if (tokenSeparator.Value == "-") { if (TokenIsNegator(context)) { context.AddToken(new Token("-", TokenType.Negator)); continue; } } context.AddToken(tokenSeparator); context.NewToken(); continue; } context.AppendToCurrentToken(c); } if (context.CurrentTokenHasValue) { context.AddToken(CreateToken(context, worksheet)); } CleanupTokens(context); return(context.Result); }
private void InitHandler(TokenizerContext context) { var parsingContext = ParsingContext.Create(); var tokenFactory = new TokenFactory(parsingContext.Configuration.FunctionRepository, null); _handler = new TokenHandler(_tokenizerContext, tokenFactory, new TokenSeparatorProvider()); }
private static void CleanupTokens(TokenizerContext context, IDictionary <string, Token> tokens) { for (int i = 0; i < context.Result.Count; i++) { var token = context.Result[i]; if (token.TokenType == TokenType.Unrecognized) { if (i < context.Result.Count - 1) { if (context.Result[i + 1].TokenType == TokenType.OpeningParenthesis) { token.TokenType = TokenType.Function; } else { token.TokenType = TokenType.NameValue; } } else { token.TokenType = TokenType.NameValue; } } else if (token.TokenType == TokenType.WorksheetName) { // use this and the following three tokens token.TokenType = TokenType.ExcelAddress; var sb = new StringBuilder(); var nToRemove = 3; if (context.Result.Count < i + nToRemove) { token.TokenType = TokenType.InvalidReference; nToRemove = context.Result.Count - i - 1; } else if (context.Result[i + 3].TokenType != TokenType.ExcelAddress) { token.TokenType = TokenType.InvalidReference; nToRemove--; } else { for (var ix = 0; ix < 4; ix++) { sb.Append(context.Result[i + ix].Value); } } token.Value = sb.ToString(); for (var ix = 0; ix < nToRemove; ix++) { context.Result.RemoveAt(i + 1); } } // Clean up leading '+' along with the following operator combinations: ++, --, +-, -+ else if ((token.TokenType == TokenType.Operator || token.TokenType == TokenType.Negator) && i < context.Result.Count - 1 && (token.Value == "+" || token.Value == "-")) { //Remove '+' from start of formula and formula arguments if (token.Value == "+" && (i == 0 || context.Result[i - 1].TokenType == TokenType.OpeningParenthesis || context.Result[i - 1].TokenType == TokenType.Comma)) { context.Result.RemoveAt(i); SetNegatorOperator(context, i, tokens); i--; continue; } var nextToken = context.Result[i + 1]; if (nextToken.TokenType == TokenType.Operator || nextToken.TokenType == TokenType.Negator) { // Remove leading '+' from operator combinations if (token.Value == "+" && (nextToken.Value == "+" || nextToken.Value == "-")) { context.Result.RemoveAt(i); SetNegatorOperator(context, i, tokens); i--; } // Remove trailing '+' from a negator operation else if (token.Value == "-" && nextToken.Value == "+") { context.Result.RemoveAt(i + 1); SetNegatorOperator(context, i, tokens); i--; } // Convert double negator operation to positive declaration else if (token.Value == "-" && nextToken.Value == "-") { context.Result.RemoveAt(i); context.Result[i] = tokens["+"]; i--; } } } } }
private static bool IsDoubleQuote(Token tokenSeparator, int formulaCharIndex, TokenizerContext context) { return tokenSeparator.TokenType == TokenType.String && formulaCharIndex + 1 < context.FormulaChars.Length && context.FormulaChars[formulaCharIndex + 1] == '\"'; }