public TokenHandler(TokenizerContext context, ITokenFactory tokenFactory, ITokenSeparatorProvider tokenProvider, TokenSeparatorHandler tokenSeparatorHandler)
{
_context = context;
_tokenFactory = tokenFactory;
_tokenProvider = tokenProvider;
_tokenSeparatorHandler = tokenSeparatorHandler;
}
public IEnumerable <Token> Tokenize(string input, string worksheet)
{
if (string.IsNullOrEmpty(input))
{
return(Enumerable.Empty <Token>());
}
// MA 1401: Ignore leading plus in formula.
input = input.TrimStart('+');
var context = new TokenizerContext(input);
var handler = new TokenHandler(context, _tokenFactory, _separatorProvider);
handler.Worksheet = worksheet;
while (handler.HasMore())
{
handler.Next();
}
if (context.CurrentTokenHasValue)
{
context.AddToken(CreateToken(context, worksheet));
}
CleanupTokens(context, _separatorProvider.Tokens);
return(context.Result);
}
private bool IsPartOfMultipleCharSeparator(TokenizerContext context, char c)
{
var lastToken = context.LastToken != null ? context.LastToken.Value : string.Empty;
return(_tokenProvider.IsOperator(lastToken) &&
_tokenProvider.IsPossibleLastPartOfMultipleCharOperator(c.ToString(CultureInfo.InvariantCulture)) &&
!context.CurrentTokenHasValue);
}
private Token CreateToken(TokenizerContext context, string worksheet)
{
if (context.CurrentToken == "-")
{
if (context.LastToken == null && context.LastToken.TokenType == TokenType.Operator)
{
return(new Token("-", TokenType.Negator));
}
}
return(_tokenFactory.Create(context.Result, context.CurrentToken, worksheet));
}
private static void HandleUnrecognizedTokens(TokenizerContext context, IDictionary <string, Token> tokens)
{
int i = 0;
while (i < context.Result.Count)
{
var token = context.Result[i];
if (token.TokenType == TokenType.Unrecognized)
{
//Check 3-D sheet reference, Note: 3-D reference with single sheet name will be seen as a single sheet name
if (i < context.Result.Count - 4 && context.Result[i + 1].TokenType == TokenType.Colon && context.Result[i + 2].TokenType == TokenType.Unrecognized &&
context.Result[i + 3].TokenType == TokenType.ExclamationMark && _sheetReferenceTokens.Contains(context.Result[i + 4].TokenType))
{
token.TokenType = TokenType.WorksheetName;
context.Result[i + 2].TokenType = TokenType.WorksheetName;
i += 4;
continue;
}
//Check regular sheet reference
if (i < context.Result.Count - 2 && context.Result[i + 1].TokenType == TokenType.ExclamationMark && _sheetReferenceTokens.Contains(context.Result[i + 2].TokenType))
{
token.TokenType = TokenType.WorksheetName;
i += 2;
continue;
}
//Check for function
if (i < context.Result.Count - 1 && context.Result[i + 1].TokenType == TokenType.OpeningParenthesis)
{
token.TokenType = TokenType.Function;
i += 2;
continue;
}
//Check for Column / Row reference
if (i < context.Result.Count - 2 && context.Result[i + 1].TokenType == TokenType.Colon && context.Result[i + 2].TokenType == TokenType.Unrecognized &&
IsColumnOrRowReference($"{token.Value}:{context.Result[i + 2].Value}"))
{
token.TokenType = TokenType.ExcelAddress;
context.Result[i + 2].TokenType = TokenType.ExcelAddress;
i += 3;
continue;
}
token.TokenType = TokenType.NameValue;
}
i++;
}
}
private static void SetNegatorOperator(TokenizerContext context, int i, IDictionary <string, Token> tokens)
{
if (context.Result[i].Value == "-" && i > 0 && (context.Result[i].TokenType == TokenType.Operator || context.Result[i].TokenType == TokenType.Negator))
{
if (TokenIsNegator(context.Result[i - 1]))
{
context.Result[i] = new Token("-", TokenType.Negator);
}
else
{
context.Result[i] = tokens["-"];
}
}
}
private static void SetNegatorOperator(TokenizerContext context, int i)
{
if (context.Result[i].Value == "-" && i > 0 && (context.Result[i].TokenType == TokenType.Operator || context.Result[i].TokenType == TokenType.Negator))
{
if (TokenIsNegator(context.Result[i - 1]))
{
context.Result[i].TokenType = TokenType.Negator;
}
else
{
context.Result[i].TokenType = TokenType.Operator;
}
}
}
public IEnumerable <Token> Tokenize(string input, string worksheet)
{
if (string.IsNullOrEmpty(input))
{
return(Enumerable.Empty <Token>());
}
// MA 1401: Ignore leading plus in formula.
input = input.TrimStart('+');
var context = new TokenizerContext(input, worksheet, _tokenFactory);
var handler = context.CreateHandler();
while (handler.HasMore())
{
handler.Next();
}
context.PostProcess();
return(context.Result);
}
private static bool TokenIsNegator(string token, TokenizerContext context)
{
if (token != "-")
{
return(false);
}
if (!context.LastToken.HasValue)
{
return(true);
}
var t = context.LastToken.Value;
return(t.TokenType == TokenType.Operator
||
t.TokenType == TokenType.OpeningParenthesis
||
t.TokenType == TokenType.Comma
||
t.TokenType == TokenType.SemiColon
||
t.TokenType == TokenType.OpeningEnumerable);
}
public IEnumerable <Token> Tokenize(string input, string worksheet)
{
if (string.IsNullOrEmpty(input))
{
return(Enumerable.Empty <Token>());
}
var context = new TokenizerContext(input);
var handler = new TokenHandler(context, _tokenFactory, _separatorProvider);
handler.Worksheet = worksheet;
while (handler.HasMore())
{
handler.Next();
}
if (context.CurrentTokenHasValue)
{
context.AddToken(CreateToken(context, worksheet));
}
HandleUnrecognizedTokens(context, _separatorProvider.Tokens);
return(context.Result);
}
public IEnumerable<Token> Tokenize(string input, string worksheet)
{
if (string.IsNullOrEmpty(input))
{
return Enumerable.Empty<Token>();
}
// MA 1401: Ignore leading plus in formula.
input = input.TrimStart('+');
var context = new TokenizerContext(input);
var handler = new TokenHandler(context, _tokenFactory, _separatorProvider);
handler.Worksheet = worksheet;
while(handler.HasMore())
{
handler.Next();
}
if (context.CurrentTokenHasValue)
{
context.AddToken(CreateToken(context, worksheet));
}
CleanupTokens(context, _separatorProvider.Tokens);
return context.Result;
}
private Token CreateToken(TokenizerContext context, string worksheet)
{
if (context.CurrentToken == "-")
{
if (context.LastToken == null && context.LastToken.TokenType == TokenType.Operator)
{
return new Token("-", TokenType.Negator);
}
}
return _tokenFactory.Create(context.Result, context.CurrentToken, worksheet);
}
private static bool TokenIsNegator(TokenizerContext context)
{
return TokenIsNegator(context.LastToken);
}
private static void SetNegatorOperator(TokenizerContext context, int i)
{
if (context.Result[i].Value == "-" && i > 0 && (context.Result[i].TokenType == TokenType.Operator || context.Result[i].TokenType == TokenType.Negator))
{
if (TokenIsNegator(context.Result[i - 1]))
{
context.Result[i].TokenType = TokenType.Negator;
}
else
{
context.Result[i].TokenType = TokenType.Operator;
}
}
}
private static void CleanupTokens(TokenizerContext context)
{
for (int i = 0; i < context.Result.Count; i++)
{
var token=context.Result[i];
if (token.TokenType == TokenType.Unrecognized)
{
if (i < context.Result.Count - 1)
{
if (context.Result[i+1].TokenType == TokenType.OpeningParenthesis)
{
token.TokenType = TokenType.Function;
}
else
{
token.TokenType = TokenType.NameValue;
}
}
else
{
token.TokenType = TokenType.NameValue;
}
}
else if ((token.TokenType == TokenType.Operator || token.TokenType == TokenType.Negator) && i < context.Result.Count - 1 &&
(token.Value=="+" || token.Value=="-"))
{
if (i > 0 && token.Value == "+") //Remove any + with an opening parenthesis before.
{
if (context.Result[i - 1].TokenType == TokenType.OpeningParenthesis)
{
context.Result.RemoveAt(i);
SetNegatorOperator(context, i);
i--;
continue;
}
}
var nextToken = context.Result[i + 1];
if (nextToken.TokenType == TokenType.Operator || nextToken.TokenType == TokenType.Negator)
{
if (token.Value == "+" && (nextToken.Value=="+" || nextToken.Value == "-"))
{
//Remove first
context.Result.RemoveAt(i);
SetNegatorOperator(context, i);
i--;
}
else if (token.Value == "-" && nextToken.Value == "+")
{
//Remove second
context.Result.RemoveAt(i+1);
SetNegatorOperator(context, i);
i--;
}
else if (token.Value == "-" && nextToken.Value == "-")
{
//Remove first and set operator to +
context.Result.RemoveAt(i);
if (i == 0)
{
context.Result.RemoveAt(i+1);
i += 2;
}
else
{
context.Result[i].TokenType = TokenType.Operator;
context.Result[i].Value = "+";
SetNegatorOperator(context, i);
i--;
}
}
}
}
}
}
public TokenHandler(TokenizerContext context, ITokenFactory tokenFactory, ITokenSeparatorProvider tokenProvider)
: this(context, tokenFactory, tokenProvider, new TokenSeparatorHandler(tokenProvider))
{
}
private static bool TokenIsNegator(TokenizerContext context)
{
return(TokenIsNegator(context.LastToken));
}
public IEnumerable<Token> Tokenize(string input, string worksheet)
{
if (string.IsNullOrEmpty(input))
{
return Enumerable.Empty<Token>();
}
// MA 1401: Ignore leading plus in formula.
input = input.TrimStart('+');
var context = new TokenizerContext(input);
for (int i = 0; i<context.FormulaChars.Length;i++)
{
var c = context.FormulaChars[i];
Token tokenSeparator;
if (CharIsTokenSeparator(c, out tokenSeparator))
{
if (context.IsInString)
{
if (tokenSeparator.TokenType == TokenType.String && i + 1 < context.FormulaChars.Length && context.FormulaChars[i + 1] == '\'')
{
i++;
context.AppendToCurrentToken(c);
continue;
}
else if(tokenSeparator.TokenType != TokenType.String)
{
context.AppendToCurrentToken(c);
continue;
}
}
if (tokenSeparator.TokenType == TokenType.OpeningBracket)
{
context.AppendToCurrentToken(c);
context.BracketCount++;
continue;
}
if (tokenSeparator.TokenType == TokenType.ClosingBracket)
{
context.AppendToCurrentToken(c);
context.BracketCount--;
continue;
}
if (context.BracketCount > 0)
{
context.AppendToCurrentToken(c);
continue;
}
// two operators in sequence could be "<=" or ">="
if (IsPartOfMultipleCharSeparator(context, c))
{
var sOp = context.LastToken.Value + c.ToString(CultureInfo.InvariantCulture);
var op = _tokenProvider.Tokens[sOp];
context.ReplaceLastToken(op);
context.NewToken();
continue;
}
if (tokenSeparator.TokenType == TokenType.String)
{
if (context.LastToken != null && context.LastToken.TokenType == TokenType.OpeningEnumerable)
{
context.AppendToCurrentToken(c);
context.ToggleIsInString();
continue;
}
else if (context.LastToken != null &&
context.LastToken.TokenType == TokenType.String &&
!context.CurrentTokenHasValue) //Added check for enumartion
{
// We are dealing with an empty string ('').
context.AddToken(new Token(string.Empty, TokenType.StringContent));
}
context.ToggleIsInString();
}
if (context.CurrentTokenHasValue)
{
context.AddToken(CreateToken(context, worksheet));
//If the a next token is an opening parantheses and the previous token is interpeted as an address or name, then the currenct token is a function
if(tokenSeparator.TokenType==TokenType.OpeningParenthesis && (context.LastToken.TokenType==TokenType.ExcelAddress || context.LastToken.TokenType==TokenType.NameValue))
{
context.LastToken.TokenType=TokenType.Function;
}
}
if (tokenSeparator.Value == "-")
{
if (TokenIsNegator(context))
{
context.AddToken(new Token("-", TokenType.Negator));
continue;
}
}
context.AddToken(tokenSeparator);
context.NewToken();
continue;
}
context.AppendToCurrentToken(c);
}
if (context.CurrentTokenHasValue)
{
context.AddToken(CreateToken(context, worksheet));
}
CleanupTokens(context);
return context.Result;
}
private static bool IsDoubleQuote(Token tokenSeparator, int formulaCharIndex, TokenizerContext context)
{
return(tokenSeparator.TokenType == TokenType.String && formulaCharIndex + 1 < context.FormulaChars.Length && context.FormulaChars[formulaCharIndex + 1] == '\"');
}
private static void CleanupTokens(TokenizerContext context, IDictionary <string, Token> tokens)
{
for (int i = 0; i < context.Result.Count; i++)
{
var token = context.Result[i];
if (token.TokenType == TokenType.Unrecognized)
{
if (i < context.Result.Count - 1)
{
if (context.Result[i + 1].TokenType == TokenType.OpeningParenthesis)
{
token.TokenType = TokenType.Function;
}
else
{
token.TokenType = TokenType.NameValue;
}
}
else
{
token.TokenType = TokenType.NameValue;
}
}
else if (token.TokenType == TokenType.WorksheetName)
{
// use this and the following three tokens
token.TokenType = TokenType.ExcelAddress;
var sb = new StringBuilder();
var nToRemove = 3;
if (context.Result.Count < i + nToRemove)
{
token.TokenType = TokenType.InvalidReference;
nToRemove = context.Result.Count - i - 1;
}
else if (context.Result[i + 3].TokenType != TokenType.ExcelAddress)
{
token.TokenType = TokenType.InvalidReference;
nToRemove--;
}
else
{
for (var ix = 0; ix < 4; ix++)
{
sb.Append(context.Result[i + ix].Value);
}
}
token.Value = sb.ToString();
for (var ix = 0; ix < nToRemove; ix++)
{
context.Result.RemoveAt(i + 1);
}
}
else if ((token.TokenType == TokenType.Operator || token.TokenType == TokenType.Negator) && i < context.Result.Count - 1 &&
(token.Value == "+" || token.Value == "-"))
{
if (i > 0 && token.Value == "+") //Remove any + with an opening parenthesis before.
{
if (context.Result[i - 1].TokenType == TokenType.OpeningParenthesis)
{
context.Result.RemoveAt(i);
SetNegatorOperator(context, i, tokens);
i--;
continue;
}
}
var nextToken = context.Result[i + 1];
if (nextToken.TokenType == TokenType.Operator || nextToken.TokenType == TokenType.Negator)
{
if (token.Value == "+" && (nextToken.Value == "+" || nextToken.Value == "-"))
{
//Remove first
context.Result.RemoveAt(i);
SetNegatorOperator(context, i, tokens);
i--;
}
else if (token.Value == "-" && nextToken.Value == "+")
{
//Remove second
context.Result.RemoveAt(i + 1);
SetNegatorOperator(context, i, tokens);
i--;
}
else if (token.Value == "-" && nextToken.Value == "-")
{
//Remove first and set operator to +
context.Result.RemoveAt(i);
if (i == 0)
{
context.Result.RemoveAt(i + 1);
i += 2;
}
else
{
//context.Result[i].TokenType = TokenType.Operator;
//context.Result[i].Value = "+";
context.Result[i] = tokens["+"];
SetNegatorOperator(context, i, tokens);
i--;
}
}
}
}
}
}
public void Init()
{
_tokenizerContext = new TokenizerContext("test");
InitHandler(_tokenizerContext);
}
private static void CleanupTokens(TokenizerContext context)
{
for (int i = 0; i < context.Result.Count; i++)
{
var token = context.Result[i];
if (token.TokenType == TokenType.Unrecognized)
{
if (i < context.Result.Count - 1)
{
if (context.Result[i + 1].TokenType == TokenType.OpeningParenthesis)
{
token.TokenType = TokenType.Function;
}
else
{
token.TokenType = TokenType.NameValue;
}
}
else
{
token.TokenType = TokenType.NameValue;
}
}
else if ((token.TokenType == TokenType.Operator || token.TokenType == TokenType.Negator) && i < context.Result.Count - 1 &&
(token.Value == "+" || token.Value == "-"))
{
if (i > 0 && token.Value == "+") //Remove any + with an opening parenthesis before.
{
if (context.Result[i - 1].TokenType == TokenType.OpeningParenthesis)
{
context.Result.RemoveAt(i);
SetNegatorOperator(context, i);
i--;
continue;
}
}
var nextToken = context.Result[i + 1];
if (nextToken.TokenType == TokenType.Operator || nextToken.TokenType == TokenType.Negator)
{
if (token.Value == "+" && (nextToken.Value == "+" || nextToken.Value == "-"))
{
//Remove first
context.Result.RemoveAt(i);
SetNegatorOperator(context, i);
i--;
}
else if (token.Value == "-" && nextToken.Value == "+")
{
//Remove second
context.Result.RemoveAt(i + 1);
SetNegatorOperator(context, i);
i--;
}
else if (token.Value == "-" && nextToken.Value == "-")
{
//Remove first and set operator to +
context.Result.RemoveAt(i);
if (i == 0)
{
context.Result.RemoveAt(i + 1);
i += 2;
}
else
{
context.Result[i].TokenType = TokenType.Operator;
context.Result[i].Value = "+";
SetNegatorOperator(context, i);
i--;
}
}
}
}
}
}
private void FixOperators(TokenizerContext context)
{
}
public TokenHandler(TokenizerContext context, ITokenFactory tokenFactory, ITokenSeparatorProvider tokenProvider)
{
_context = context;
_tokenFactory = tokenFactory;
_tokenProvider = tokenProvider;
}
private void FixOperators(TokenizerContext context)
{
}
private bool IsPartOfMultipleCharSeparator(TokenizerContext context, char c)
{
var lastToken = context.LastToken != null ? context.LastToken.Value : string.Empty;
return _tokenProvider.IsOperator(lastToken)
&& _tokenProvider.IsPossibleLastPartOfMultipleCharOperator(c.ToString(CultureInfo.InvariantCulture))
&& !context.CurrentTokenHasValue;
}
private static void SetNegatorOperator(TokenizerContext context, int i, IDictionary<string, Token> tokens)
{
if (context.Result[i].Value == "-" && i > 0 && (context.Result[i].TokenType == TokenType.Operator || context.Result[i].TokenType == TokenType.Negator))
{
if (TokenIsNegator(context.Result[i - 1]))
{
context.Result[i] = new Token("-", TokenType.Negator);
}
else
{
context.Result[i] = tokens["-"];
}
}
}
private static void CleanupTokens(TokenizerContext context, IDictionary<string, Token> tokens)
{
for (int i = 0; i < context.Result.Count; i++)
{
var token=context.Result[i];
if (token.TokenType == TokenType.Unrecognized)
{
if (i < context.Result.Count - 1)
{
if (context.Result[i+1].TokenType == TokenType.OpeningParenthesis)
{
token.TokenType = TokenType.Function;
}
else
{
token.TokenType = TokenType.NameValue;
}
}
else
{
token.TokenType = TokenType.NameValue;
}
}
else if(token.TokenType == TokenType.WorksheetName){
// use this and the following three tokens
token.TokenType = TokenType.ExcelAddress;
var sb = new StringBuilder();
var nToRemove = 3;
if (context.Result.Count < i + nToRemove)
{
token.TokenType = TokenType.InvalidReference;
nToRemove = context.Result.Count - i - 1;
}
else if(context.Result[i + 3].TokenType != TokenType.ExcelAddress)
{
token.TokenType = TokenType.InvalidReference;
nToRemove--;
}
else
{
for (var ix = 0; ix < 4; ix++)
{
sb.Append(context.Result[i + ix].Value);
}
}
token.Value = sb.ToString();
for(var ix = 0; ix < nToRemove; ix++)
{
context.Result.RemoveAt(i + 1);
}
}
// Clean up leading '+' along with the following operator combinations: ++, --, +-, -+
else if ((token.TokenType == TokenType.Operator || token.TokenType == TokenType.Negator) && i < context.Result.Count - 1 &&
(token.Value=="+" || token.Value=="-"))
{
//Remove '+' from start of formula and formula arguments
if (token.Value == "+" && (i == 0 || context.Result[i - 1].TokenType == TokenType.OpeningParenthesis || context.Result[i - 1].TokenType == TokenType.Comma))
{
context.Result.RemoveAt(i);
SetNegatorOperator(context, i, tokens);
i--;
continue;
}
var nextToken = context.Result[i + 1];
if (nextToken.TokenType == TokenType.Operator || nextToken.TokenType == TokenType.Negator)
{
// Remove leading '+' from operator combinations
if (token.Value == "+" && (nextToken.Value=="+" || nextToken.Value == "-"))
{
context.Result.RemoveAt(i);
SetNegatorOperator(context, i, tokens);
i--;
}
// Remove trailing '+' from a negator operation
else if (token.Value == "-" && nextToken.Value == "+")
{
context.Result.RemoveAt(i+1);
SetNegatorOperator(context, i, tokens);
i--;
}
// Convert double negator operation to positive declaration
else if (token.Value == "-" && nextToken.Value == "-")
{
context.Result.RemoveAt(i);
context.Result[i] = tokens["+"];
i--;
}
}
}
}
}
private static void CleanupTokens(TokenizerContext context, IDictionary<string, Token> tokens)
{
for (int i = 0; i < context.Result.Count; i++)
{
var token=context.Result[i];
if (token.TokenType == TokenType.Unrecognized)
{
if (i < context.Result.Count - 1)
{
if (context.Result[i+1].TokenType == TokenType.OpeningParenthesis)
{
token.TokenType = TokenType.Function;
}
else
{
token.TokenType = TokenType.NameValue;
}
}
else
{
token.TokenType = TokenType.NameValue;
}
}
else if(token.TokenType == TokenType.WorksheetName){
// use this and the following three tokens
token.TokenType = TokenType.ExcelAddress;
var sb = new StringBuilder();
var nToRemove = 3;
if (context.Result.Count < i + nToRemove)
{
token.TokenType = TokenType.InvalidReference;
nToRemove = context.Result.Count - i - 1;
}
else if(context.Result[i + 3].TokenType != TokenType.ExcelAddress)
{
token.TokenType = TokenType.InvalidReference;
nToRemove--;
}
else
{
for (var ix = 0; ix < 4; ix++)
{
sb.Append(context.Result[i + ix].Value);
}
}
token.Value = sb.ToString();
for(var ix = 0; ix < nToRemove; ix++)
{
context.Result.RemoveAt(i + 1);
}
}
else if ((token.TokenType == TokenType.Operator || token.TokenType == TokenType.Negator) && i < context.Result.Count - 1 &&
(token.Value=="+" || token.Value=="-"))
{
if (i > 0 && token.Value == "+") //Remove any + with an opening parenthesis before.
{
if (context.Result[i - 1].TokenType == TokenType.OpeningParenthesis)
{
context.Result.RemoveAt(i);
SetNegatorOperator(context, i, tokens);
i--;
continue;
}
}
var nextToken = context.Result[i + 1];
if (nextToken.TokenType == TokenType.Operator || nextToken.TokenType == TokenType.Negator)
{
if (token.Value == "+" && (nextToken.Value=="+" || nextToken.Value == "-"))
{
//Remove first
context.Result.RemoveAt(i);
SetNegatorOperator(context, i, tokens);
i--;
}
else if (token.Value == "-" && nextToken.Value == "+")
{
//Remove second
context.Result.RemoveAt(i+1);
SetNegatorOperator(context, i, tokens);
i--;
}
else if (token.Value == "-" && nextToken.Value == "-")
{
//Remove first and set operator to +
context.Result.RemoveAt(i);
if (i == 0)
{
context.Result.RemoveAt(i+1);
i += 2;
}
else
{
//context.Result[i].TokenType = TokenType.Operator;
//context.Result[i].Value = "+";
context.Result[i] = tokens["+"];
SetNegatorOperator(context, i, tokens);
i--;
}
}
}
}
}
}
public IEnumerable <Token> Tokenize(string input, string worksheet)
{
if (string.IsNullOrEmpty(input))
{
return(Enumerable.Empty <Token>());
}
// MA 1401: Ignore leading plus in formula.
input = input.TrimStart('+');
var context = new TokenizerContext(input);
for (int i = 0; i < context.FormulaChars.Length; i++)
{
var c = context.FormulaChars[i];
Token tokenSeparator;
if (CharIsTokenSeparator(c, out tokenSeparator))
{
if (context.IsInString)
{
if (tokenSeparator.TokenType == TokenType.String && i + 1 < context.FormulaChars.Length && context.FormulaChars[i + 1] == '\'')
{
i++;
context.AppendToCurrentToken(c);
continue;
}
else if (tokenSeparator.TokenType != TokenType.String)
{
context.AppendToCurrentToken(c);
continue;
}
}
if (tokenSeparator.TokenType == TokenType.OpeningBracket)
{
context.AppendToCurrentToken(c);
context.BracketCount++;
continue;
}
if (tokenSeparator.TokenType == TokenType.ClosingBracket)
{
context.AppendToCurrentToken(c);
context.BracketCount--;
continue;
}
if (context.BracketCount > 0)
{
context.AppendToCurrentToken(c);
continue;
}
// two operators in sequence could be "<=" or ">="
if (IsPartOfMultipleCharSeparator(context, c))
{
var sOp = context.LastToken.Value + c.ToString(CultureInfo.InvariantCulture);
var op = _tokenProvider.Tokens[sOp];
context.ReplaceLastToken(op);
context.NewToken();
continue;
}
if (tokenSeparator.TokenType == TokenType.String)
{
if (context.LastToken != null && context.LastToken.TokenType == TokenType.OpeningEnumerable)
{
context.AppendToCurrentToken(c);
context.ToggleIsInString();
continue;
}
else if (context.LastToken != null &&
context.LastToken.TokenType == TokenType.String &&
!context.CurrentTokenHasValue) //Added check for enumartion
{
// We are dealing with an empty string ('').
context.AddToken(new Token(string.Empty, TokenType.StringContent));
}
context.ToggleIsInString();
}
if (context.CurrentTokenHasValue)
{
context.AddToken(CreateToken(context, worksheet));
//If the a next token is an opening parantheses and the previous token is interpeted as an address or name, then the currenct token is a function
if (tokenSeparator.TokenType == TokenType.OpeningParenthesis && (context.LastToken.TokenType == TokenType.ExcelAddress || context.LastToken.TokenType == TokenType.NameValue))
{
context.LastToken.TokenType = TokenType.Function;
}
}
if (tokenSeparator.Value == "-")
{
if (TokenIsNegator(context))
{
context.AddToken(new Token("-", TokenType.Negator));
continue;
}
}
context.AddToken(tokenSeparator);
context.NewToken();
continue;
}
context.AppendToCurrentToken(c);
}
if (context.CurrentTokenHasValue)
{
context.AddToken(CreateToken(context, worksheet));
}
CleanupTokens(context);
return(context.Result);
}
private void InitHandler(TokenizerContext context)
{
var parsingContext = ParsingContext.Create();
var tokenFactory = new TokenFactory(parsingContext.Configuration.FunctionRepository, null);
_handler = new TokenHandler(_tokenizerContext, tokenFactory, new TokenSeparatorProvider());
}
private static void CleanupTokens(TokenizerContext context, IDictionary <string, Token> tokens)
{
for (int i = 0; i < context.Result.Count; i++)
{
var token = context.Result[i];
if (token.TokenType == TokenType.Unrecognized)
{
if (i < context.Result.Count - 1)
{
if (context.Result[i + 1].TokenType == TokenType.OpeningParenthesis)
{
token.TokenType = TokenType.Function;
}
else
{
token.TokenType = TokenType.NameValue;
}
}
else
{
token.TokenType = TokenType.NameValue;
}
}
else if (token.TokenType == TokenType.WorksheetName)
{
// use this and the following three tokens
token.TokenType = TokenType.ExcelAddress;
var sb = new StringBuilder();
var nToRemove = 3;
if (context.Result.Count < i + nToRemove)
{
token.TokenType = TokenType.InvalidReference;
nToRemove = context.Result.Count - i - 1;
}
else if (context.Result[i + 3].TokenType != TokenType.ExcelAddress)
{
token.TokenType = TokenType.InvalidReference;
nToRemove--;
}
else
{
for (var ix = 0; ix < 4; ix++)
{
sb.Append(context.Result[i + ix].Value);
}
}
token.Value = sb.ToString();
for (var ix = 0; ix < nToRemove; ix++)
{
context.Result.RemoveAt(i + 1);
}
}
// Clean up leading '+' along with the following operator combinations: ++, --, +-, -+
else if ((token.TokenType == TokenType.Operator || token.TokenType == TokenType.Negator) && i < context.Result.Count - 1 &&
(token.Value == "+" || token.Value == "-"))
{
//Remove '+' from start of formula and formula arguments
if (token.Value == "+" && (i == 0 || context.Result[i - 1].TokenType == TokenType.OpeningParenthesis || context.Result[i - 1].TokenType == TokenType.Comma))
{
context.Result.RemoveAt(i);
SetNegatorOperator(context, i, tokens);
i--;
continue;
}
var nextToken = context.Result[i + 1];
if (nextToken.TokenType == TokenType.Operator || nextToken.TokenType == TokenType.Negator)
{
// Remove leading '+' from operator combinations
if (token.Value == "+" && (nextToken.Value == "+" || nextToken.Value == "-"))
{
context.Result.RemoveAt(i);
SetNegatorOperator(context, i, tokens);
i--;
}
// Remove trailing '+' from a negator operation
else if (token.Value == "-" && nextToken.Value == "+")
{
context.Result.RemoveAt(i + 1);
SetNegatorOperator(context, i, tokens);
i--;
}
// Convert double negator operation to positive declaration
else if (token.Value == "-" && nextToken.Value == "-")
{
context.Result.RemoveAt(i);
context.Result[i] = tokens["+"];
i--;
}
}
}
}
}
private static bool IsDoubleQuote(Token tokenSeparator, int formulaCharIndex, TokenizerContext context)
{
return tokenSeparator.TokenType == TokenType.String && formulaCharIndex + 1 < context.FormulaChars.Length && context.FormulaChars[formulaCharIndex + 1] == '\"';
}
