public TokenList Cut(int startPos)
{
var res = new TokenList();
for (int i = startPos; i < this.Count(); i++)
{
res.Add(this[i]);
}
return(res);
}
/// <summary>
/// Inserts token between two elements in list after `startIndex` if their types are `type1` and `type2` and their values are not in `exclude` list
/// </summary>
public void InsertBetween(Token.Type type1, Token.Type type2, Token token)
{
if (this.Count() == 0)
{
return;
}
TokenList newList = new TokenList();
for (int i = 0; i < this.Count() - 1; i++)
{
newList.Add(this[i]);
if (this[i].type == type1 && this[i + 1].type == type2)
{
newList.Add(token);
}
}
newList.Add(this.Last());
Clear();
AddRange(newList);
}
TokenList ConcatTwoTokens(TokenList list, Token token1, Token token2, Token replacement)
{
if (list.Count() == 0)
{
return(list);
}
TokenList newList = new TokenList();
newList.Add(list[0]);
for (int i = 1; i < list.Count(); i++)
{
if (list[i - 1] == token1 && list[i] == token2)
{
newList[newList.Count() - 1] = replacement;
}
else
{
newList.Add(list[i]);
}
}
return(newList);
}
public TokenList GenerateTokens()
{
TokenList tokenList = new TokenList();
if (string.IsNullOrEmpty(expr))
{
return(tokenList);
}
expr += '\n'; // to avoid checking end of expr
int idx = 0;
while (idx < expr.Length)
{
if (char.IsLetter(expr[idx]))
{
Token token = ParseVariable(expr, ref idx);
if (keywordList.Contains(token.value))
{
token.type = Token.Type.KEYWORD;
}
else if (functionList.Contains(token.value))
{
token.type = Token.Type.SYSTEM_FUNCTION;
}
else if (constList.Contains(token.value))
{
token.type = Token.Type.SYSTEM_CONST;
}
tokenList.Add(token);
}
else if (char.IsDigit(expr[idx]))
{
tokenList.Add(ParseNumeric(expr, ref idx));
}
else if (operatorList.Contains(expr[idx]))
{
tokenList.Add(new Token(Token.Type.MATH_OP, expr[idx].ToString()));
idx++;
}
else if ("{([|".Contains(expr[idx]))
{
tokenList.Add(new Token(Token.Type.BRACKET_OPEN, expr[idx].ToString()));
idx++;
}
else if ("})]|".Contains(expr[idx]))
{
tokenList.Add(new Token(Token.Type.BRACKET_CLOSE, expr[idx].ToString()));
idx++;
}
else if (char.IsWhiteSpace(expr[idx]))
{
idx++;
}
else
{
throw new LexisException(expr[idx].ToString());
}
}
return(ConcatTokens(tokenList));
}