private void MakeNumericToken()
{
tokenStart = pointer;
while (LexerHelper.IsNumeric(currChar))
{
token += currChar;
MoveNext();
}
tokenStart--;
AddTokenAndResetTemp(TokenType.Numeric);
}
private bool IsEndOfToken()
{
if (LexerHelper.IsTokenSeparator(currChar))
{
return(true);
}
if (LexerHelper.IsCorrectIdentifier(token) != LexerHelper.IsCorrectIdentifier(currChar))
{
return(true);
}
if (LexerHelper.IsNumeric(token) != LexerHelper.IsNumeric(currChar))
{
return(true);
}
if (GetTokenType(token + currChar) != TokenType.Operation &&
GetTokenType(token) == TokenType.Operation &&
GetTokenType(currChar) == TokenType.Operation)
{
return(true);
}
return(false);
}
public List <Token> SplitToTokenList(string scriptText)
{
scriptText += ";";
tokens = new List <Token>();
script = scriptText;
pointer = -1;
token = "";
length = script.Length;
while (MoveNext())
{
if (token == "/" && currChar == '/')
{
MakeComment();
}
if (token == "\"")
{
MakeTextToken();
}
if (LexerHelper.IsNumeric(token))
{
MakeNumericToken();
}
if (LexerHelper.IsTokenSeparator(token) || (token.Length > 0 && IsEndOfToken()))
{
AddTokenAndResetTemp(GetTokenType(token));
}
token += currChar;
}
return(tokens);
}