TokenList ConcatTwoTokens(TokenList list, Token token1, Token token2, Token replacement)
{
if (list.Count() == 0)
{
return(list);
}
TokenList newList = new TokenList();
newList.Add(list[0]);
for (int i = 1; i < list.Count(); i++)
{
if (list[i - 1] == token1 && list[i] == token2)
{
newList[newList.Count() - 1] = replacement;
}
else
{
newList.Add(list[i]);
}
}
return(newList);
}
public TokenList GenerateTokens()
{
TokenList tokenList = new TokenList();
if (string.IsNullOrEmpty(expr))
{
return(tokenList);
}
expr += '\n'; // to avoid checking end of expr
int idx = 0;
while (idx < expr.Length)
{
if (char.IsLetter(expr[idx]))
{
Token token = ParseVariable(expr, ref idx);
if (keywordList.Contains(token.value))
{
token.type = Token.Type.KEYWORD;
}
else if (functionList.Contains(token.value))
{
token.type = Token.Type.SYSTEM_FUNCTION;
}
else if (constList.Contains(token.value))
{
token.type = Token.Type.SYSTEM_CONST;
}
tokenList.Add(token);
}
else if (char.IsDigit(expr[idx]))
{
tokenList.Add(ParseNumeric(expr, ref idx));
}
else if (operatorList.Contains(expr[idx]))
{
tokenList.Add(new Token(Token.Type.MATH_OP, expr[idx].ToString()));
idx++;
}
else if ("{([|".Contains(expr[idx]))
{
tokenList.Add(new Token(Token.Type.BRACKET_OPEN, expr[idx].ToString()));
idx++;
}
else if ("})]|".Contains(expr[idx]))
{
tokenList.Add(new Token(Token.Type.BRACKET_CLOSE, expr[idx].ToString()));
idx++;
}
else if (char.IsWhiteSpace(expr[idx]))
{
idx++;
}
else
{
throw new LexisException(expr[idx].ToString());
}
}
return(ConcatTokens(tokenList));
}