public TokenList Cut(int startPos) { var res = new TokenList(); for (int i = startPos; i < this.Count(); i++) { res.Add(this[i]); } return(res); }
/// <summary> /// Inserts token between two elements in list after `startIndex` if their types are `type1` and `type2` and their values are not in `exclude` list /// </summary> public void InsertBetween(Token.Type type1, Token.Type type2, Token token) { if (this.Count() == 0) { return; } TokenList newList = new TokenList(); for (int i = 0; i < this.Count() - 1; i++) { newList.Add(this[i]); if (this[i].type == type1 && this[i + 1].type == type2) { newList.Add(token); } } newList.Add(this.Last()); Clear(); AddRange(newList); }
TokenList ConcatTwoTokens(TokenList list, Token token1, Token token2, Token replacement) { if (list.Count() == 0) { return(list); } TokenList newList = new TokenList(); newList.Add(list[0]); for (int i = 1; i < list.Count(); i++) { if (list[i - 1] == token1 && list[i] == token2) { newList[newList.Count() - 1] = replacement; } else { newList.Add(list[i]); } } return(newList); }
public TokenList GenerateTokens() { TokenList tokenList = new TokenList(); if (string.IsNullOrEmpty(expr)) { return(tokenList); } expr += '\n'; // to avoid checking end of expr int idx = 0; while (idx < expr.Length) { if (char.IsLetter(expr[idx])) { Token token = ParseVariable(expr, ref idx); if (keywordList.Contains(token.value)) { token.type = Token.Type.KEYWORD; } else if (functionList.Contains(token.value)) { token.type = Token.Type.SYSTEM_FUNCTION; } else if (constList.Contains(token.value)) { token.type = Token.Type.SYSTEM_CONST; } tokenList.Add(token); } else if (char.IsDigit(expr[idx])) { tokenList.Add(ParseNumeric(expr, ref idx)); } else if (operatorList.Contains(expr[idx])) { tokenList.Add(new Token(Token.Type.MATH_OP, expr[idx].ToString())); idx++; } else if ("{([|".Contains(expr[idx])) { tokenList.Add(new Token(Token.Type.BRACKET_OPEN, expr[idx].ToString())); idx++; } else if ("})]|".Contains(expr[idx])) { tokenList.Add(new Token(Token.Type.BRACKET_CLOSE, expr[idx].ToString())); idx++; } else if (char.IsWhiteSpace(expr[idx])) { idx++; } else { throw new LexisException(expr[idx].ToString()); } } return(ConcatTokens(tokenList)); }