private void MakeNumericToken() { tokenStart = pointer; while (LexerHelper.IsNumeric(currChar)) { token += currChar; MoveNext(); } tokenStart--; AddTokenAndResetTemp(TokenType.Numeric); }
private bool IsEndOfToken() { if (LexerHelper.IsTokenSeparator(currChar)) { return(true); } if (LexerHelper.IsCorrectIdentifier(token) != LexerHelper.IsCorrectIdentifier(currChar)) { return(true); } if (LexerHelper.IsNumeric(token) != LexerHelper.IsNumeric(currChar)) { return(true); } if (GetTokenType(token + currChar) != TokenType.Operation && GetTokenType(token) == TokenType.Operation && GetTokenType(currChar) == TokenType.Operation) { return(true); } return(false); }
public List <Token> SplitToTokenList(string scriptText) { scriptText += ";"; tokens = new List <Token>(); script = scriptText; pointer = -1; token = ""; length = script.Length; while (MoveNext()) { if (token == "/" && currChar == '/') { MakeComment(); } if (token == "\"") { MakeTextToken(); } if (LexerHelper.IsNumeric(token)) { MakeNumericToken(); } if (LexerHelper.IsTokenSeparator(token) || (token.Length > 0 && IsEndOfToken())) { AddTokenAndResetTemp(GetTokenType(token)); } token += currChar; } return(tokens); }